2024-11-25 03:56:49,910 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-25 03:56:49,932 main DEBUG Took 0.019171 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 03:56:49,933 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 03:56:49,933 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 03:56:49,934 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 03:56:49,936 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:56:49,945 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 03:56:49,959 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:56:49,960 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:56:49,961 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:56:49,962 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:56:49,963 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:56:49,963 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:56:49,964 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:56:49,964 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:56:49,965 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:56:49,965 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:56:49,966 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:56:49,967 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:56:49,968 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:56:49,968 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:56:49,969 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:56:49,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:56:49,970 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:56:49,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:56:49,971 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:56:49,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:56:49,972 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:56:49,972 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:56:49,973 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:56:49,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:56:49,974 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:56:49,974 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 03:56:49,976 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:56:49,977 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 03:56:49,979 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 03:56:49,980 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 03:56:49,981 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 03:56:49,981 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 03:56:49,990 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 03:56:49,993 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 03:56:49,995 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 03:56:49,995 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 03:56:49,996 main DEBUG createAppenders(={Console}) 2024-11-25 03:56:49,997 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-25 03:56:49,997 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-25 03:56:49,997 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-25 03:56:49,998 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 03:56:49,998 main DEBUG OutputStream closed 2024-11-25 03:56:49,999 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 03:56:49,999 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 03:56:49,999 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-25 03:56:50,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 03:56:50,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 03:56:50,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 03:56:50,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 03:56:50,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 03:56:50,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 03:56:50,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 03:56:50,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 03:56:50,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 03:56:50,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 03:56:50,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 03:56:50,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 03:56:50,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 03:56:50,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 03:56:50,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 03:56:50,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 03:56:50,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 03:56:50,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 03:56:50,105 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 03:56:50,105 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-25 03:56:50,106 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 03:56:50,106 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-25T03:56:50,123 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-25 03:56:50,127 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 03:56:50,127 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T03:56:50,413 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7 2024-11-25T03:56:50,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T03:56:50,462 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/cluster_0e4f7757-e78d-eac9-4af1-03483e95706d, deleteOnExit=true 2024-11-25T03:56:50,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T03:56:50,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/test.cache.data in system properties and HBase conf 2024-11-25T03:56:50,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T03:56:50,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/hadoop.log.dir in system properties and HBase conf 2024-11-25T03:56:50,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T03:56:50,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T03:56:50,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T03:56:50,584 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T03:56:50,712 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:56:50,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T03:56:50,718 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T03:56:50,719 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T03:56:50,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T03:56:50,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T03:56:50,721 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T03:56:50,721 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T03:56:50,722 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T03:56:50,723 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T03:56:50,723 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/nfs.dump.dir in system properties and HBase conf 2024-11-25T03:56:50,724 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/java.io.tmpdir in system properties and HBase conf 2024-11-25T03:56:50,724 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T03:56:50,725 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T03:56:50,726 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T03:56:52,128 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T03:56:52,206 INFO [Time-limited test {}] log.Log(170): Logging initialized @3098ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T03:56:52,289 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:56:52,360 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:56:52,400 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:56:52,400 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:56:52,402 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T03:56:52,416 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:56:52,419 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:56:52,420 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:56:52,634 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/java.io.tmpdir/jetty-localhost-34093-hadoop-hdfs-3_4_1-tests_jar-_-any-12026343730370931186/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T03:56:52,641 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:34093} 2024-11-25T03:56:52,642 INFO [Time-limited test {}] server.Server(415): Started @3536ms 2024-11-25T03:56:53,215 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:56:53,227 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:56:53,228 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:56:53,229 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:56:53,229 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T03:56:53,230 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:56:53,231 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:56:53,369 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/java.io.tmpdir/jetty-localhost-45459-hadoop-hdfs-3_4_1-tests_jar-_-any-8851068025784389256/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:56:53,370 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:45459} 2024-11-25T03:56:53,370 INFO [Time-limited test {}] server.Server(415): Started @4264ms 2024-11-25T03:56:53,427 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T03:56:54,619 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/cluster_0e4f7757-e78d-eac9-4af1-03483e95706d/data/data1/current/BP-823003919-172.17.0.2-1732507011402/current, will proceed with Du for space computation calculation, 2024-11-25T03:56:54,619 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/cluster_0e4f7757-e78d-eac9-4af1-03483e95706d/data/data2/current/BP-823003919-172.17.0.2-1732507011402/current, will proceed with Du for space computation calculation, 2024-11-25T03:56:54,649 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T03:56:54,703 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb17896e717c744c with lease ID 0x243d9a58d9597cda: Processing first storage report for DS-a3000841-6322-470e-a0ea-6d9ce3b534d2 from datanode DatanodeRegistration(127.0.0.1:36505, datanodeUuid=08cc274d-cd7c-426f-a1e2-53d88e7c2d05, infoPort=36411, infoSecurePort=0, ipcPort=46003, storageInfo=lv=-57;cid=testClusterID;nsid=1359641431;c=1732507011402) 2024-11-25T03:56:54,704 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb17896e717c744c with lease ID 0x243d9a58d9597cda: from storage DS-a3000841-6322-470e-a0ea-6d9ce3b534d2 node DatanodeRegistration(127.0.0.1:36505, datanodeUuid=08cc274d-cd7c-426f-a1e2-53d88e7c2d05, infoPort=36411, infoSecurePort=0, ipcPort=46003, storageInfo=lv=-57;cid=testClusterID;nsid=1359641431;c=1732507011402), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-25T03:56:54,705 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb17896e717c744c with lease ID 0x243d9a58d9597cda: Processing first storage report for DS-ed0ef2f9-311e-4a4f-870a-f72f38754994 from datanode DatanodeRegistration(127.0.0.1:36505, datanodeUuid=08cc274d-cd7c-426f-a1e2-53d88e7c2d05, infoPort=36411, infoSecurePort=0, ipcPort=46003, storageInfo=lv=-57;cid=testClusterID;nsid=1359641431;c=1732507011402) 2024-11-25T03:56:54,705 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb17896e717c744c with lease ID 0x243d9a58d9597cda: from storage DS-ed0ef2f9-311e-4a4f-870a-f72f38754994 node DatanodeRegistration(127.0.0.1:36505, datanodeUuid=08cc274d-cd7c-426f-a1e2-53d88e7c2d05, infoPort=36411, infoSecurePort=0, ipcPort=46003, storageInfo=lv=-57;cid=testClusterID;nsid=1359641431;c=1732507011402), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T03:56:54,781 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7 2024-11-25T03:56:55,016 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/cluster_0e4f7757-e78d-eac9-4af1-03483e95706d/zookeeper_0, clientPort=50296, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/cluster_0e4f7757-e78d-eac9-4af1-03483e95706d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/cluster_0e4f7757-e78d-eac9-4af1-03483e95706d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T03:56:55,027 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50296 2024-11-25T03:56:55,039 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:56:55,042 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:56:55,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741825_1001 (size=7) 2024-11-25T03:56:55,687 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc with version=8 2024-11-25T03:56:55,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/hbase-staging 2024-11-25T03:56:55,787 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T03:56:56,054 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d73269e94d72:0 server-side Connection retries=9 2024-11-25T03:56:56,065 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:56:56,066 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:56:56,072 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:56:56,072 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:56:56,073 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:56:56,252 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T03:56:56,322 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T03:56:56,332 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T03:56:56,336 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:56:56,363 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 87232 (auto-detected) 2024-11-25T03:56:56,364 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-25T03:56:56,385 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38513 2024-11-25T03:56:56,410 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38513 connecting to ZooKeeper ensemble=127.0.0.1:50296 2024-11-25T03:56:56,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:385130x0, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:56:56,539 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38513-0x101702312f90000 connected 2024-11-25T03:56:56,632 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:56:56,636 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:56:56,648 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:56:56,652 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc, hbase.cluster.distributed=false 2024-11-25T03:56:56,681 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:56:56,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38513 2024-11-25T03:56:56,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38513 2024-11-25T03:56:56,688 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38513 2024-11-25T03:56:56,688 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38513 2024-11-25T03:56:56,688 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38513 2024-11-25T03:56:56,795 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d73269e94d72:0 server-side Connection retries=9 2024-11-25T03:56:56,797 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:56:56,797 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:56:56,798 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:56:56,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:56:56,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:56:56,801 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:56:56,803 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:56:56,804 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33579 2024-11-25T03:56:56,807 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33579 connecting to ZooKeeper ensemble=127.0.0.1:50296 2024-11-25T03:56:56,808 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:56:56,811 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:56:56,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:335790x0, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:56:56,847 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:335790x0, quorum=127.0.0.1:50296, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:56:56,848 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33579-0x101702312f90001 connected 2024-11-25T03:56:56,851 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:56:56,859 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:56:56,861 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T03:56:56,866 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:56:56,867 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33579 2024-11-25T03:56:56,867 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33579 2024-11-25T03:56:56,868 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33579 2024-11-25T03:56:56,868 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33579 2024-11-25T03:56:56,868 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33579 2024-11-25T03:56:56,888 DEBUG [M:0;d73269e94d72:38513 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d73269e94d72:38513 2024-11-25T03:56:56,890 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d73269e94d72,38513,1732507015876 2024-11-25T03:56:56,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:56:56,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:56:56,906 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d73269e94d72,38513,1732507015876 2024-11-25T03:56:56,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T03:56:56,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:56:56,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:56:56,936 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T03:56:56,939 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d73269e94d72,38513,1732507015876 from backup master directory 2024-11-25T03:56:56,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d73269e94d72,38513,1732507015876 2024-11-25T03:56:56,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:56:56,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:56:56,952 WARN [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:56:56,952 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d73269e94d72,38513,1732507015876 2024-11-25T03:56:56,955 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T03:56:56,956 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T03:56:57,015 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/hbase.id] with ID: eff93331-474f-4917-9eb5-14d08e29d57a 2024-11-25T03:56:57,015 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/.tmp/hbase.id 2024-11-25T03:56:57,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741826_1002 (size=42) 2024-11-25T03:56:57,428 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/.tmp/hbase.id]:[hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/hbase.id] 2024-11-25T03:56:57,477 INFO [master/d73269e94d72:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:56:57,483 INFO [master/d73269e94d72:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T03:56:57,503 INFO [master/d73269e94d72:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-25T03:56:57,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:56:57,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:56:57,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741827_1003 (size=196) 2024-11-25T03:56:58,001 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:56:58,004 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T03:56:58,021 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:56:58,026 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:56:58,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741828_1004 (size=1189) 2024-11-25T03:56:58,498 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store 2024-11-25T03:56:58,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741829_1005 (size=34) 2024-11-25T03:56:58,924 INFO [master/d73269e94d72:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T03:56:58,928 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:56:58,929 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:56:58,929 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:56:58,930 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:56:58,931 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:56:58,931 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:56:58,932 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:56:58,933 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732507018929Disabling compacts and flushes for region at 1732507018929Disabling writes for close at 1732507018931 (+2 ms)Writing region close event to WAL at 1732507018932 (+1 ms)Closed at 1732507018932 2024-11-25T03:56:58,936 WARN [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/.initializing 2024-11-25T03:56:58,936 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/WALs/d73269e94d72,38513,1732507015876 2024-11-25T03:56:58,945 INFO [master/d73269e94d72:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:56:58,961 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C38513%2C1732507015876, suffix=, logDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/WALs/d73269e94d72,38513,1732507015876, archiveDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/oldWALs, maxLogs=10 2024-11-25T03:56:58,985 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/WALs/d73269e94d72,38513,1732507015876/d73269e94d72%2C38513%2C1732507015876.1732507018966, exclude list is [], retry=0 2024-11-25T03:56:59,006 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36505,DS-a3000841-6322-470e-a0ea-6d9ce3b534d2,DISK] 2024-11-25T03:56:59,010 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T03:56:59,051 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/WALs/d73269e94d72,38513,1732507015876/d73269e94d72%2C38513%2C1732507015876.1732507018966 2024-11-25T03:56:59,052 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36411:36411)] 2024-11-25T03:56:59,053 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:56:59,054 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:56:59,058 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:56:59,060 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:56:59,102 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:56:59,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T03:56:59,131 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:56:59,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:56:59,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:56:59,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T03:56:59,138 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:56:59,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:56:59,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:56:59,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T03:56:59,143 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:56:59,145 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:56:59,145 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:56:59,148 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T03:56:59,148 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:56:59,150 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:56:59,150 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:56:59,155 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:56:59,156 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:56:59,162 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:56:59,163 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:56:59,168 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:56:59,173 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:56:59,178 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:56:59,180 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75402479, jitterRate=0.12358449399471283}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:56:59,189 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732507019075Initializing all the Stores at 1732507019078 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732507019078Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507019079 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507019079Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507019080 (+1 ms)Cleaning up temporary data from old regions at 1732507019164 (+84 ms)Region opened successfully at 1732507019189 (+25 ms) 2024-11-25T03:56:59,191 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T03:56:59,227 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fbe833b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:56:59,260 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T03:56:59,271 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T03:56:59,271 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T03:56:59,274 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:56:59,276 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-25T03:56:59,281 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-25T03:56:59,281 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T03:56:59,310 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T03:56:59,321 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T03:56:59,372 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T03:56:59,375 INFO [master/d73269e94d72:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T03:56:59,377 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T03:56:59,387 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T03:56:59,390 INFO [master/d73269e94d72:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T03:56:59,394 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T03:56:59,404 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T03:56:59,405 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T03:56:59,418 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T03:56:59,438 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T03:56:59,450 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T03:56:59,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:56:59,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:56:59,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:56:59,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:56:59,465 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d73269e94d72,38513,1732507015876, sessionid=0x101702312f90000, setting cluster-up flag (Was=false) 2024-11-25T03:56:59,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:56:59,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:56:59,530 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T03:56:59,533 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d73269e94d72,38513,1732507015876 2024-11-25T03:56:59,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:56:59,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:56:59,587 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T03:56:59,589 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d73269e94d72,38513,1732507015876 2024-11-25T03:56:59,597 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T03:56:59,675 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(746): ClusterId : eff93331-474f-4917-9eb5-14d08e29d57a 2024-11-25T03:56:59,677 DEBUG [RS:0;d73269e94d72:33579 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:56:59,677 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:56:59,688 INFO [master/d73269e94d72:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T03:56:59,690 DEBUG [RS:0;d73269e94d72:33579 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:56:59,690 DEBUG [RS:0;d73269e94d72:33579 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:56:59,694 INFO [master/d73269e94d72:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T03:56:59,701 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d73269e94d72,38513,1732507015876 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T03:56:59,704 DEBUG [RS:0;d73269e94d72:33579 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:56:59,705 DEBUG [RS:0;d73269e94d72:33579 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1829c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:56:59,709 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:56:59,709 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:56:59,709 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:56:59,709 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:56:59,710 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d73269e94d72:0, corePoolSize=10, maxPoolSize=10 2024-11-25T03:56:59,710 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:56:59,710 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:56:59,710 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:56:59,712 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732507049712 2024-11-25T03:56:59,714 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T03:56:59,715 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T03:56:59,716 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:56:59,717 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T03:56:59,719 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T03:56:59,720 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T03:56:59,720 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T03:56:59,720 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T03:56:59,723 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:56:59,723 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d73269e94d72:33579 2024-11-25T03:56:59,724 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T03:56:59,728 INFO [RS:0;d73269e94d72:33579 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:56:59,725 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:56:59,728 INFO [RS:0;d73269e94d72:33579 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:56:59,728 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:56:59,730 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T03:56:59,731 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T03:56:59,732 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,38513,1732507015876 with port=33579, startcode=1732507016762 2024-11-25T03:56:59,732 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T03:56:59,740 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T03:56:59,740 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T03:56:59,743 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732507019742,5,FailOnTimeoutGroup] 2024-11-25T03:56:59,744 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732507019743,5,FailOnTimeoutGroup] 2024-11-25T03:56:59,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741831_1007 (size=1321) 2024-11-25T03:56:59,744 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:56:59,744 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T03:56:59,745 DEBUG [RS:0;d73269e94d72:33579 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:56:59,746 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T03:56:59,746 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T03:56:59,747 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T03:56:59,747 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc 2024-11-25T03:56:59,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741832_1008 (size=32) 2024-11-25T03:56:59,784 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37959, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:56:59,791 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38513 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d73269e94d72,33579,1732507016762 2024-11-25T03:56:59,793 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38513 {}] master.ServerManager(517): Registering regionserver=d73269e94d72,33579,1732507016762 2024-11-25T03:56:59,811 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc 2024-11-25T03:56:59,811 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39991 2024-11-25T03:56:59,811 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:56:59,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:56:59,826 DEBUG [RS:0;d73269e94d72:33579 {}] zookeeper.ZKUtil(111): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d73269e94d72,33579,1732507016762 2024-11-25T03:56:59,826 WARN [RS:0;d73269e94d72:33579 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:56:59,826 INFO [RS:0;d73269e94d72:33579 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:56:59,826 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/WALs/d73269e94d72,33579,1732507016762 2024-11-25T03:56:59,828 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d73269e94d72,33579,1732507016762] 2024-11-25T03:56:59,856 INFO [RS:0;d73269e94d72:33579 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:56:59,875 INFO [RS:0;d73269e94d72:33579 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:56:59,880 INFO [RS:0;d73269e94d72:33579 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:56:59,880 INFO [RS:0;d73269e94d72:33579 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:56:59,881 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:56:59,887 INFO [RS:0;d73269e94d72:33579 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:56:59,888 INFO [RS:0;d73269e94d72:33579 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:56:59,888 DEBUG [RS:0;d73269e94d72:33579 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:56:59,888 DEBUG [RS:0;d73269e94d72:33579 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:56:59,889 DEBUG [RS:0;d73269e94d72:33579 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:56:59,889 DEBUG [RS:0;d73269e94d72:33579 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:56:59,889 DEBUG [RS:0;d73269e94d72:33579 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:56:59,889 DEBUG [RS:0;d73269e94d72:33579 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:56:59,889 DEBUG [RS:0;d73269e94d72:33579 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:56:59,889 DEBUG [RS:0;d73269e94d72:33579 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:56:59,889 DEBUG [RS:0;d73269e94d72:33579 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:56:59,890 DEBUG [RS:0;d73269e94d72:33579 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:56:59,890 DEBUG [RS:0;d73269e94d72:33579 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:56:59,890 DEBUG [RS:0;d73269e94d72:33579 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:56:59,890 DEBUG [RS:0;d73269e94d72:33579 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:56:59,890 DEBUG [RS:0;d73269e94d72:33579 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:56:59,897 INFO [RS:0;d73269e94d72:33579 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:56:59,897 INFO [RS:0;d73269e94d72:33579 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:56:59,898 INFO [RS:0;d73269e94d72:33579 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:56:59,898 INFO [RS:0;d73269e94d72:33579 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:56:59,898 INFO [RS:0;d73269e94d72:33579 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:56:59,898 INFO [RS:0;d73269e94d72:33579 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,33579,1732507016762-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:56:59,921 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:56:59,923 INFO [RS:0;d73269e94d72:33579 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,33579,1732507016762-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:56:59,923 INFO [RS:0;d73269e94d72:33579 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:56:59,924 INFO [RS:0;d73269e94d72:33579 {}] regionserver.Replication(171): d73269e94d72,33579,1732507016762 started 2024-11-25T03:56:59,943 INFO [RS:0;d73269e94d72:33579 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:56:59,944 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1482): Serving as d73269e94d72,33579,1732507016762, RpcServer on d73269e94d72/172.17.0.2:33579, sessionid=0x101702312f90001 2024-11-25T03:56:59,945 DEBUG [RS:0;d73269e94d72:33579 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:56:59,945 DEBUG [RS:0;d73269e94d72:33579 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d73269e94d72,33579,1732507016762 2024-11-25T03:56:59,945 DEBUG [RS:0;d73269e94d72:33579 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,33579,1732507016762' 2024-11-25T03:56:59,945 DEBUG [RS:0;d73269e94d72:33579 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:56:59,946 DEBUG [RS:0;d73269e94d72:33579 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:56:59,947 DEBUG [RS:0;d73269e94d72:33579 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:56:59,947 DEBUG [RS:0;d73269e94d72:33579 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:56:59,947 DEBUG [RS:0;d73269e94d72:33579 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d73269e94d72,33579,1732507016762 2024-11-25T03:56:59,948 DEBUG [RS:0;d73269e94d72:33579 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,33579,1732507016762' 2024-11-25T03:56:59,948 DEBUG [RS:0;d73269e94d72:33579 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:56:59,948 DEBUG [RS:0;d73269e94d72:33579 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:56:59,949 DEBUG [RS:0;d73269e94d72:33579 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:56:59,949 INFO [RS:0;d73269e94d72:33579 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:56:59,949 INFO [RS:0;d73269e94d72:33579 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:57:00,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:57:00,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:57:00,166 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:57:00,167 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:00,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:57:00,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:57:00,171 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:57:00,172 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:00,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:57:00,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:57:00,176 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:57:00,177 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:00,178 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:57:00,178 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:57:00,181 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:57:00,181 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:00,182 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:57:00,182 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:57:00,184 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740 2024-11-25T03:57:00,185 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740 2024-11-25T03:57:00,188 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:57:00,188 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:57:00,189 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:57:00,192 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:57:00,197 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:57:00,198 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60293057, jitterRate=-0.10156343877315521}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:57:00,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732507020160Initializing all the Stores at 1732507020162 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732507020163 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732507020163Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507020163Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732507020163Cleaning up temporary data from old regions at 1732507020188 (+25 ms)Region opened successfully at 1732507020201 (+13 ms) 2024-11-25T03:57:00,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:57:00,201 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:57:00,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:57:00,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:57:00,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:57:00,203 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:57:00,203 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732507020201Disabling compacts and flushes for region at 1732507020201Disabling writes for close at 1732507020201Writing region close event to WAL at 1732507020202 (+1 ms)Closed at 1732507020203 (+1 ms) 2024-11-25T03:57:00,206 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:57:00,206 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T03:57:00,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T03:57:00,220 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:57:00,223 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T03:57:00,374 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:01,375 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:02,376 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:03,377 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:04,378 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:05,378 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:06,318 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T03:57:06,319 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T03:57:06,379 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:07,380 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:08,380 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:09,381 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:10,382 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:11,382 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:12,383 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:13,384 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:14,385 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:15,386 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:16,387 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:17,387 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:18,388 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:19,389 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:20,390 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:21,391 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:22,391 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:23,392 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:24,393 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:24,777 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:57:25,393 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:26,394 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:27,395 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:28,396 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:29,396 WARN [d73269e94d72:38513 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:57:29,954 INFO [RS:0;d73269e94d72:33579 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:57:29,958 INFO [RS:0;d73269e94d72:33579 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C33579%2C1732507016762, suffix=, logDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/WALs/d73269e94d72,33579,1732507016762, archiveDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/oldWALs, maxLogs=32 2024-11-25T03:57:29,973 DEBUG [RS:0;d73269e94d72:33579 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/WALs/d73269e94d72,33579,1732507016762/d73269e94d72%2C33579%2C1732507016762.1732507049961, exclude list is [], retry=0 2024-11-25T03:57:29,978 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36505,DS-a3000841-6322-470e-a0ea-6d9ce3b534d2,DISK] 2024-11-25T03:57:29,981 INFO [RS:0;d73269e94d72:33579 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/WALs/d73269e94d72,33579,1732507016762/d73269e94d72%2C33579%2C1732507016762.1732507049961 2024-11-25T03:57:29,981 DEBUG [RS:0;d73269e94d72:33579 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36411:36411)] 2024-11-25T03:57:30,150 DEBUG [d73269e94d72:38513 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-25T03:57:30,162 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:57:30,168 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33579,1732507016762, state=OPENING 2024-11-25T03:57:30,212 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T03:57:30,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:57:30,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:57:30,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:57:30,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:57:30,229 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:57:30,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33579,1732507016762}] 2024-11-25T03:57:30,410 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:57:30,414 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60445, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:57:30,426 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T03:57:30,427 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:57:30,428 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T03:57:30,431 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C33579%2C1732507016762.meta, suffix=.meta, logDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/WALs/d73269e94d72,33579,1732507016762, archiveDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/oldWALs, maxLogs=32 2024-11-25T03:57:30,446 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/WALs/d73269e94d72,33579,1732507016762/d73269e94d72%2C33579%2C1732507016762.meta.1732507050433.meta, exclude list is [], retry=0 2024-11-25T03:57:30,449 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36505,DS-a3000841-6322-470e-a0ea-6d9ce3b534d2,DISK] 2024-11-25T03:57:30,452 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/WALs/d73269e94d72,33579,1732507016762/d73269e94d72%2C33579%2C1732507016762.meta.1732507050433.meta 2024-11-25T03:57:30,453 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36411:36411)] 2024-11-25T03:57:30,453 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:57:30,455 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T03:57:30,457 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T03:57:30,461 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T03:57:30,465 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T03:57:30,466 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:57:30,466 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T03:57:30,466 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T03:57:30,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:57:30,470 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:57:30,471 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:30,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:57:30,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:57:30,474 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:57:30,474 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:30,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:57:30,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:57:30,476 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:57:30,477 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:30,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:57:30,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:57:30,479 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:57:30,479 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:30,480 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:57:30,480 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:57:30,482 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740 2024-11-25T03:57:30,485 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740 2024-11-25T03:57:30,487 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:57:30,488 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:57:30,488 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:57:30,491 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:57:30,493 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73213801, jitterRate=0.09097065031528473}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:57:30,493 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T03:57:30,494 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732507050466Writing region info on filesystem at 1732507050467 (+1 ms)Initializing all the Stores at 1732507050468 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732507050468Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732507050469 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507050469Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732507050469Cleaning up temporary data from old regions at 1732507050488 (+19 ms)Running coprocessor post-open hooks at 1732507050493 (+5 ms)Region opened successfully at 1732507050494 (+1 ms) 2024-11-25T03:57:30,501 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732507050403 2024-11-25T03:57:30,513 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T03:57:30,514 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T03:57:30,516 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:57:30,518 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33579,1732507016762, state=OPEN 2024-11-25T03:57:30,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:57:30,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:57:30,613 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:57:30,613 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:57:30,613 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d73269e94d72,33579,1732507016762 2024-11-25T03:57:30,619 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T03:57:30,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33579,1732507016762 in 381 msec 2024-11-25T03:57:30,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T03:57:30,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.4090 sec 2024-11-25T03:57:30,629 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:57:30,629 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T03:57:30,646 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:57:30,647 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33579,1732507016762, seqNum=-1] 2024-11-25T03:57:30,664 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:57:30,666 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58983, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:57:30,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 31.0550 sec 2024-11-25T03:57:30,689 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732507050688, completionTime=-1 2024-11-25T03:57:30,692 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-25T03:57:30,693 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:57:30,717 INFO [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-25T03:57:30,717 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732507110717 2024-11-25T03:57:30,717 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732507170717 2024-11-25T03:57:30,717 INFO [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-25T03:57:30,719 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,38513,1732507015876-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:57:30,719 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,38513,1732507015876-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:57:30,720 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,38513,1732507015876-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:57:30,721 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d73269e94d72:38513, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:57:30,721 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T03:57:30,722 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T03:57:30,729 DEBUG [master/d73269e94d72:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T03:57:30,747 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.794sec 2024-11-25T03:57:30,748 INFO [master/d73269e94d72:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T03:57:30,750 INFO [master/d73269e94d72:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T03:57:30,750 INFO [master/d73269e94d72:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T03:57:30,751 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T03:57:30,751 INFO [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T03:57:30,752 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,38513,1732507015876-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:57:30,752 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,38513,1732507015876-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T03:57:30,761 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T03:57:30,762 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T03:57:30,762 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,38513,1732507015876-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:57:30,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f875e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:57:30,821 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T03:57:30,821 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T03:57:30,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d73269e94d72,38513,-1 for getting cluster id 2024-11-25T03:57:30,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:57:30,834 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'eff93331-474f-4917-9eb5-14d08e29d57a' 2024-11-25T03:57:30,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:57:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "eff93331-474f-4917-9eb5-14d08e29d57a" 2024-11-25T03:57:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f79f4b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:57:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d73269e94d72,38513,-1] 2024-11-25T03:57:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:57:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:57:30,845 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:57:30,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7ec3ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:57:30,849 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:57:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33579,1732507016762, seqNum=-1] 2024-11-25T03:57:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:57:30,861 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:57:30,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d73269e94d72,38513,1732507015876 2024-11-25T03:57:30,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb7c601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:57:30,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d73269e94d72,38513,-1 for getting cluster id 2024-11-25T03:57:30,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:57:30,883 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'eff93331-474f-4917-9eb5-14d08e29d57a' 2024-11-25T03:57:30,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:57:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "eff93331-474f-4917-9eb5-14d08e29d57a" 2024-11-25T03:57:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44627131, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:57:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d73269e94d72,38513,-1] 2024-11-25T03:57:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:57:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:57:30,887 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:57:30,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c9e7cd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:57:30,901 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9740 2024-11-25T03:57:30,907 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:57:30,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d73269e94d72,38513,1732507015876 2024-11-25T03:57:30,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@53adb2ba 2024-11-25T03:57:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:57:30,917 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:57:30,923 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:57:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-25T03:57:30,952 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:57:30,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-25T03:57:30,955 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:30,957 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:57:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:57:30,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741835_1011 (size=428) 2024-11-25T03:57:31,373 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7cc8c925e00abed8a7bf0e5438157cb7, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc 2024-11-25T03:57:31,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741836_1012 (size=79) 2024-11-25T03:57:31,784 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:57:31,785 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 7cc8c925e00abed8a7bf0e5438157cb7, disabling compactions & flushes 2024-11-25T03:57:31,785 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. 2024-11-25T03:57:31,785 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. 2024-11-25T03:57:31,785 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. after waiting 0 ms 2024-11-25T03:57:31,785 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. 2024-11-25T03:57:31,785 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. 2024-11-25T03:57:31,785 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7cc8c925e00abed8a7bf0e5438157cb7: Waiting for close lock at 1732507051784Disabling compacts and flushes for region at 1732507051784Disabling writes for close at 1732507051785 (+1 ms)Writing region close event to WAL at 1732507051785Closed at 1732507051785 2024-11-25T03:57:31,789 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:57:31,795 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732507051790"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732507051790"}]},"ts":"1732507051790"} 2024-11-25T03:57:31,799 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:57:31,801 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:57:31,804 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507051802"}]},"ts":"1732507051802"} 2024-11-25T03:57:31,808 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-25T03:57:31,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=7cc8c925e00abed8a7bf0e5438157cb7, ASSIGN}] 2024-11-25T03:57:31,812 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=7cc8c925e00abed8a7bf0e5438157cb7, ASSIGN 2024-11-25T03:57:31,814 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=7cc8c925e00abed8a7bf0e5438157cb7, ASSIGN; state=OFFLINE, location=d73269e94d72,33579,1732507016762; forceNewPlan=false, retain=false 2024-11-25T03:57:31,966 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7cc8c925e00abed8a7bf0e5438157cb7, regionState=OPENING, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:57:31,973 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=7cc8c925e00abed8a7bf0e5438157cb7, ASSIGN because future has completed 2024-11-25T03:57:31,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cc8c925e00abed8a7bf0e5438157cb7, server=d73269e94d72,33579,1732507016762}] 2024-11-25T03:57:31,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:57:32,135 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. 2024-11-25T03:57:32,136 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7cc8c925e00abed8a7bf0e5438157cb7, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:57:32,136 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 7cc8c925e00abed8a7bf0e5438157cb7 2024-11-25T03:57:32,137 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:57:32,137 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7cc8c925e00abed8a7bf0e5438157cb7 2024-11-25T03:57:32,137 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7cc8c925e00abed8a7bf0e5438157cb7 2024-11-25T03:57:32,139 INFO [StoreOpener-7cc8c925e00abed8a7bf0e5438157cb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7cc8c925e00abed8a7bf0e5438157cb7 2024-11-25T03:57:32,142 INFO [StoreOpener-7cc8c925e00abed8a7bf0e5438157cb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cc8c925e00abed8a7bf0e5438157cb7 columnFamilyName testFamily 2024-11-25T03:57:32,142 DEBUG [StoreOpener-7cc8c925e00abed8a7bf0e5438157cb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:32,144 INFO [StoreOpener-7cc8c925e00abed8a7bf0e5438157cb7-1 {}] regionserver.HStore(327): Store=7cc8c925e00abed8a7bf0e5438157cb7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:57:32,144 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7cc8c925e00abed8a7bf0e5438157cb7 2024-11-25T03:57:32,146 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7cc8c925e00abed8a7bf0e5438157cb7 2024-11-25T03:57:32,146 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7cc8c925e00abed8a7bf0e5438157cb7 2024-11-25T03:57:32,147 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7cc8c925e00abed8a7bf0e5438157cb7 2024-11-25T03:57:32,147 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7cc8c925e00abed8a7bf0e5438157cb7 2024-11-25T03:57:32,150 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7cc8c925e00abed8a7bf0e5438157cb7 2024-11-25T03:57:32,154 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7cc8c925e00abed8a7bf0e5438157cb7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:57:32,156 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7cc8c925e00abed8a7bf0e5438157cb7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63812442, jitterRate=-0.04912051558494568}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:57:32,156 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7cc8c925e00abed8a7bf0e5438157cb7 2024-11-25T03:57:32,157 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7cc8c925e00abed8a7bf0e5438157cb7: Running coprocessor pre-open hook at 1732507052137Writing region info on filesystem at 1732507052137Initializing all the Stores at 1732507052139 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507052139Cleaning up temporary data from old regions at 1732507052147 (+8 ms)Running coprocessor post-open hooks at 1732507052156 (+9 ms)Region opened successfully at 1732507052157 (+1 ms) 2024-11-25T03:57:32,159 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7., pid=6, masterSystemTime=1732507052128 2024-11-25T03:57:32,162 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. 2024-11-25T03:57:32,163 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. 2024-11-25T03:57:32,164 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7cc8c925e00abed8a7bf0e5438157cb7, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:57:32,167 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cc8c925e00abed8a7bf0e5438157cb7, server=d73269e94d72,33579,1732507016762 because future has completed 2024-11-25T03:57:32,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T03:57:32,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7cc8c925e00abed8a7bf0e5438157cb7, server=d73269e94d72,33579,1732507016762 in 195 msec 2024-11-25T03:57:32,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T03:57:32,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=7cc8c925e00abed8a7bf0e5438157cb7, ASSIGN in 364 msec 2024-11-25T03:57:32,181 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:57:32,181 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507052181"}]},"ts":"1732507052181"} 2024-11-25T03:57:32,184 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-25T03:57:32,186 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:57:32,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2440 sec 2024-11-25T03:57:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:57:33,996 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-25T03:57:33,996 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-25T03:57:33,999 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:57:34,006 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-25T03:57:34,006 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:57:34,007 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-25T03:57:34,009 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:57:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33579,1732507016762, seqNum=-1] 2024-11-25T03:57:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:57:34,014 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:57:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7., hostname=d73269e94d72,33579,1732507016762, seqNum=2] 2024-11-25T03:57:34,036 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:57:34,039 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-25T03:57:34,044 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-25T03:57:34,050 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-25T03:57:34,052 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-25T03:57:34,053 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3839764233650700280 2024-11-25T03:57:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60454 deadline: 1732507057052, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:57:34,079 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-25T03:57:34,080 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-25T03:57:34,081 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:57:34,083 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:57:34,083 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3839764233650700278 2024-11-25T03:57:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60454 deadline: 1732507057082, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:57:34,087 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:57:34,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-3839764233650700277 2024-11-25T03:57:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60454 deadline: 1732507057087, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:57:34,094 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:57:34,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-3839764233650700276 2024-11-25T03:57:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60454 deadline: 1732507057094, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:57:34,100 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:57:34,110 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/d73269e94d72:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711893171_22 at /127.0.0.1:40514 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=166 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9671 (was 9740) 2024-11-25T03:57:34,118 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=166, ProcessCount=11, AvailableMemoryMB=9671 2024-11-25T03:57:34,122 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:57:34,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-25T03:57:34,126 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:57:34,126 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:34,126 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-25T03:57:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:57:34,129 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:57:34,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741837_1013 (size=414) 2024-11-25T03:57:34,540 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2d3b244c1794e17d8de0df895f56030d, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc 2024-11-25T03:57:34,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741838_1014 (size=65) 2024-11-25T03:57:34,951 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:57:34,952 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 2d3b244c1794e17d8de0df895f56030d, disabling compactions & flushes 2024-11-25T03:57:34,952 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. 2024-11-25T03:57:34,952 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. 2024-11-25T03:57:34,952 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. after waiting 0 ms 2024-11-25T03:57:34,952 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. 2024-11-25T03:57:34,952 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. 2024-11-25T03:57:34,952 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2d3b244c1794e17d8de0df895f56030d: Waiting for close lock at 1732507054951Disabling compacts and flushes for region at 1732507054951Disabling writes for close at 1732507054952 (+1 ms)Writing region close event to WAL at 1732507054952Closed at 1732507054952 2024-11-25T03:57:34,956 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:57:34,956 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732507054956"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732507054956"}]},"ts":"1732507054956"} 2024-11-25T03:57:34,960 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:57:34,962 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:57:34,962 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507054962"}]},"ts":"1732507054962"} 2024-11-25T03:57:34,966 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-25T03:57:34,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=2d3b244c1794e17d8de0df895f56030d, ASSIGN}] 2024-11-25T03:57:34,969 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=2d3b244c1794e17d8de0df895f56030d, ASSIGN 2024-11-25T03:57:34,970 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=2d3b244c1794e17d8de0df895f56030d, ASSIGN; state=OFFLINE, location=d73269e94d72,33579,1732507016762; forceNewPlan=false, retain=false 2024-11-25T03:57:35,122 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=2d3b244c1794e17d8de0df895f56030d, regionState=OPENING, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:57:35,128 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=2d3b244c1794e17d8de0df895f56030d, ASSIGN because future has completed 2024-11-25T03:57:35,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d3b244c1794e17d8de0df895f56030d, server=d73269e94d72,33579,1732507016762}] 2024-11-25T03:57:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:57:35,292 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. 2024-11-25T03:57:35,293 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d3b244c1794e17d8de0df895f56030d, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:57:35,293 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 2d3b244c1794e17d8de0df895f56030d 2024-11-25T03:57:35,294 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:57:35,294 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 2d3b244c1794e17d8de0df895f56030d 2024-11-25T03:57:35,294 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 2d3b244c1794e17d8de0df895f56030d 2024-11-25T03:57:35,296 INFO [StoreOpener-2d3b244c1794e17d8de0df895f56030d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2d3b244c1794e17d8de0df895f56030d 2024-11-25T03:57:35,298 INFO [StoreOpener-2d3b244c1794e17d8de0df895f56030d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d3b244c1794e17d8de0df895f56030d columnFamilyName testFamily 2024-11-25T03:57:35,298 DEBUG [StoreOpener-2d3b244c1794e17d8de0df895f56030d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:35,299 INFO [StoreOpener-2d3b244c1794e17d8de0df895f56030d-1 {}] regionserver.HStore(327): Store=2d3b244c1794e17d8de0df895f56030d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:57:35,299 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 2d3b244c1794e17d8de0df895f56030d 2024-11-25T03:57:35,300 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScanner/2d3b244c1794e17d8de0df895f56030d 2024-11-25T03:57:35,301 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScanner/2d3b244c1794e17d8de0df895f56030d 2024-11-25T03:57:35,301 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 2d3b244c1794e17d8de0df895f56030d 2024-11-25T03:57:35,301 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 2d3b244c1794e17d8de0df895f56030d 2024-11-25T03:57:35,304 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 2d3b244c1794e17d8de0df895f56030d 2024-11-25T03:57:35,308 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScanner/2d3b244c1794e17d8de0df895f56030d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:57:35,309 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 2d3b244c1794e17d8de0df895f56030d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67652022, jitterRate=0.008093684911727905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:57:35,309 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d3b244c1794e17d8de0df895f56030d 2024-11-25T03:57:35,311 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 2d3b244c1794e17d8de0df895f56030d: Running coprocessor pre-open hook at 1732507055294Writing region info on filesystem at 1732507055294Initializing all the Stores at 1732507055296 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507055296Cleaning up temporary data from old regions at 1732507055301 (+5 ms)Running coprocessor post-open hooks at 1732507055309 (+8 ms)Region opened successfully at 1732507055311 (+2 ms) 2024-11-25T03:57:35,312 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d., pid=9, masterSystemTime=1732507055284 2024-11-25T03:57:35,316 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. 2024-11-25T03:57:35,316 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. 2024-11-25T03:57:35,317 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=2d3b244c1794e17d8de0df895f56030d, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:57:35,320 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d3b244c1794e17d8de0df895f56030d, server=d73269e94d72,33579,1732507016762 because future has completed 2024-11-25T03:57:35,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-25T03:57:35,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d3b244c1794e17d8de0df895f56030d, server=d73269e94d72,33579,1732507016762 in 194 msec 2024-11-25T03:57:35,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T03:57:35,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=2d3b244c1794e17d8de0df895f56030d, ASSIGN in 359 msec 2024-11-25T03:57:35,331 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:57:35,331 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507055331"}]},"ts":"1732507055331"} 2024-11-25T03:57:35,334 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-25T03:57:35,335 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:57:35,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2130 sec 2024-11-25T03:57:36,006 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:57:36,035 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-25T03:57:36,037 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T03:57:36,038 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-25T03:57:36,317 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T03:57:36,318 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T03:57:36,319 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:57:36,319 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T03:57:36,320 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-25T03:57:36,320 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T03:57:36,321 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-25T03:57:36,321 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-25T03:57:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:57:37,156 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-25T03:57:37,156 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-25T03:57:37,157 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:57:37,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-25T03:57:37,166 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:57:37,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-25T03:57:37,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d., hostname=d73269e94d72,33579,1732507016762, seqNum=2] 2024-11-25T03:57:37,181 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:57:37,181 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-25T03:57:37,185 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-25T03:57:37,186 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-25T03:57:43,197 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d., hostname=d73269e94d72,33579,1732507016762, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d., hostname=d73269e94d72,33579,1732507016762, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d73269e94d72:33579 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6009ms, rpcTimeout=5999ms 2024-11-25T03:57:43,197 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d., hostname=d73269e94d72,33579,1732507016762, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6009ms, rpcTimeout=5999ms 2024-11-25T03:57:43,198 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d., hostname=d73269e94d72,33579,1732507016762, seqNum=2 from cache 2024-11-25T03:57:43,206 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=169 (was 166) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9578 (was 9671) 2024-11-25T03:57:43,213 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=169, ProcessCount=11, AvailableMemoryMB=9578 2024-11-25T03:57:43,215 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:57:43,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-25T03:57:43,219 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:57:43,219 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:43,219 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-25T03:57:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-25T03:57:43,221 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:57:43,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741839_1015 (size=412) 2024-11-25T03:57:43,632 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fee51fafcdd578c2ae0ea3c13d1a4565, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc 2024-11-25T03:57:43,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741840_1016 (size=63) 2024-11-25T03:57:43,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-25T03:57:44,045 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:57:44,046 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing fee51fafcdd578c2ae0ea3c13d1a4565, disabling compactions & flushes 2024-11-25T03:57:44,046 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. 2024-11-25T03:57:44,046 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. 2024-11-25T03:57:44,046 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. after waiting 0 ms 2024-11-25T03:57:44,046 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. 2024-11-25T03:57:44,047 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. 2024-11-25T03:57:44,047 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for fee51fafcdd578c2ae0ea3c13d1a4565: Waiting for close lock at 1732507064046Disabling compacts and flushes for region at 1732507064046Disabling writes for close at 1732507064046Writing region close event to WAL at 1732507064046Closed at 1732507064047 (+1 ms) 2024-11-25T03:57:44,050 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:57:44,051 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732507064050"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732507064050"}]},"ts":"1732507064050"} 2024-11-25T03:57:44,055 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:57:44,057 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:57:44,057 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507064057"}]},"ts":"1732507064057"} 2024-11-25T03:57:44,060 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-25T03:57:44,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=fee51fafcdd578c2ae0ea3c13d1a4565, ASSIGN}] 2024-11-25T03:57:44,063 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=fee51fafcdd578c2ae0ea3c13d1a4565, ASSIGN 2024-11-25T03:57:44,066 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=fee51fafcdd578c2ae0ea3c13d1a4565, ASSIGN; state=OFFLINE, location=d73269e94d72,33579,1732507016762; forceNewPlan=false, retain=false 2024-11-25T03:57:44,217 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=fee51fafcdd578c2ae0ea3c13d1a4565, regionState=OPENING, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:57:44,226 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=fee51fafcdd578c2ae0ea3c13d1a4565, ASSIGN because future has completed 2024-11-25T03:57:44,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure fee51fafcdd578c2ae0ea3c13d1a4565, server=d73269e94d72,33579,1732507016762}] 2024-11-25T03:57:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-25T03:57:44,386 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. 2024-11-25T03:57:44,387 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => fee51fafcdd578c2ae0ea3c13d1a4565, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:57:44,387 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:57:44,387 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:57:44,387 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:57:44,387 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:57:44,389 INFO [StoreOpener-fee51fafcdd578c2ae0ea3c13d1a4565-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:57:44,392 INFO [StoreOpener-fee51fafcdd578c2ae0ea3c13d1a4565-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fee51fafcdd578c2ae0ea3c13d1a4565 columnFamilyName testFamily 2024-11-25T03:57:44,392 DEBUG [StoreOpener-fee51fafcdd578c2ae0ea3c13d1a4565-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:44,393 INFO [StoreOpener-fee51fafcdd578c2ae0ea3c13d1a4565-1 {}] regionserver.HStore(327): Store=fee51fafcdd578c2ae0ea3c13d1a4565/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:57:44,393 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:57:44,395 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNextAsync/fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:57:44,396 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNextAsync/fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:57:44,396 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:57:44,396 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:57:44,399 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:57:44,403 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNextAsync/fee51fafcdd578c2ae0ea3c13d1a4565/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:57:44,403 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened fee51fafcdd578c2ae0ea3c13d1a4565; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65754118, jitterRate=-0.020187288522720337}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:57:44,404 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:57:44,404 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for fee51fafcdd578c2ae0ea3c13d1a4565: Running coprocessor pre-open hook at 1732507064387Writing region info on filesystem at 1732507064387Initializing all the Stores at 1732507064389 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507064389Cleaning up temporary data from old regions at 1732507064396 (+7 ms)Running coprocessor post-open hooks at 1732507064404 (+8 ms)Region opened successfully at 1732507064404 2024-11-25T03:57:44,406 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565., pid=12, masterSystemTime=1732507064381 2024-11-25T03:57:44,408 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. 2024-11-25T03:57:44,409 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. 2024-11-25T03:57:44,410 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=fee51fafcdd578c2ae0ea3c13d1a4565, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:57:44,413 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure fee51fafcdd578c2ae0ea3c13d1a4565, server=d73269e94d72,33579,1732507016762 because future has completed 2024-11-25T03:57:44,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-25T03:57:44,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure fee51fafcdd578c2ae0ea3c13d1a4565, server=d73269e94d72,33579,1732507016762 in 189 msec 2024-11-25T03:57:44,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-25T03:57:44,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=fee51fafcdd578c2ae0ea3c13d1a4565, ASSIGN in 357 msec 2024-11-25T03:57:44,422 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:57:44,422 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507064422"}]},"ts":"1732507064422"} 2024-11-25T03:57:44,425 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-25T03:57:44,426 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:57:44,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2100 sec 2024-11-25T03:57:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-25T03:57:46,255 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-25T03:57:46,255 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-25T03:57:46,255 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:57:46,260 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-25T03:57:46,261 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:57:46,261 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-25T03:57:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565., hostname=d73269e94d72,33579,1732507016762, seqNum=2] 2024-11-25T03:57:46,280 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:57:46,281 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-25T03:57:46,285 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-25T03:57:46,288 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-25T03:57:46,289 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-25T03:57:46,290 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-25T03:57:46,317 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-25T03:57:46,317 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T03:57:48,039 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-25T03:57:49,917 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:57:52,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3839764233650700267' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-25T03:57:52,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33579 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:60454 deadline: 1732507078294, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3839764233650700267' does not exist 2024-11-25T03:57:52,298 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to d73269e94d72,33579,1732507016762 for closing scanner id = -3839764233650700267 for fee51fafcdd578c2ae0ea3c13d1a4565 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3839764233650700267' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:57:52,301 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=165 (was 169), ProcessCount=11 (was 11), AvailableMemoryMB=9371 (was 9578) 2024-11-25T03:57:52,310 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=165, ProcessCount=11, AvailableMemoryMB=9370 2024-11-25T03:57:52,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:57:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-25T03:57:52,317 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:57:52,317 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:52,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-25T03:57:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:57:52,319 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:57:52,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741841_1017 (size=423) 2024-11-25T03:57:52,739 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8fa582fb6c7a21404f4927f1ac2befef, NAME => 'testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc 2024-11-25T03:57:52,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741842_1018 (size=74) 2024-11-25T03:57:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-25T03:57:53,156 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:57:53,156 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 8fa582fb6c7a21404f4927f1ac2befef, disabling compactions & flushes 2024-11-25T03:57:53,157 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. 2024-11-25T03:57:53,157 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. 2024-11-25T03:57:53,157 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. after waiting 0 ms 2024-11-25T03:57:53,157 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. 2024-11-25T03:57:53,157 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. 2024-11-25T03:57:53,158 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8fa582fb6c7a21404f4927f1ac2befef: Waiting for close lock at 1732507073156Disabling compacts and flushes for region at 1732507073156Disabling writes for close at 1732507073157 (+1 ms)Writing region close event to WAL at 1732507073157Closed at 1732507073157 2024-11-25T03:57:53,163 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:57:53,163 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732507073163"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732507073163"}]},"ts":"1732507073163"} 2024-11-25T03:57:53,168 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:57:53,171 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:57:53,171 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507073171"}]},"ts":"1732507073171"} 2024-11-25T03:57:53,174 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-25T03:57:53,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=8fa582fb6c7a21404f4927f1ac2befef, ASSIGN}] 2024-11-25T03:57:53,176 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=8fa582fb6c7a21404f4927f1ac2befef, ASSIGN 2024-11-25T03:57:53,177 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=8fa582fb6c7a21404f4927f1ac2befef, ASSIGN; state=OFFLINE, location=d73269e94d72,33579,1732507016762; forceNewPlan=false, retain=false 2024-11-25T03:57:53,328 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=8fa582fb6c7a21404f4927f1ac2befef, regionState=OPENING, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:57:53,331 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=8fa582fb6c7a21404f4927f1ac2befef, ASSIGN because future has completed 2024-11-25T03:57:53,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fa582fb6c7a21404f4927f1ac2befef, server=d73269e94d72,33579,1732507016762}] 2024-11-25T03:57:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:57:53,495 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. 2024-11-25T03:57:53,495 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 8fa582fb6c7a21404f4927f1ac2befef, NAME => 'testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:57:53,495 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 8fa582fb6c7a21404f4927f1ac2befef 2024-11-25T03:57:53,495 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:57:53,496 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 8fa582fb6c7a21404f4927f1ac2befef 2024-11-25T03:57:53,496 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 8fa582fb6c7a21404f4927f1ac2befef 2024-11-25T03:57:53,498 INFO [StoreOpener-8fa582fb6c7a21404f4927f1ac2befef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8fa582fb6c7a21404f4927f1ac2befef 2024-11-25T03:57:53,502 INFO [StoreOpener-8fa582fb6c7a21404f4927f1ac2befef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8fa582fb6c7a21404f4927f1ac2befef columnFamilyName testFamily 2024-11-25T03:57:53,503 DEBUG [StoreOpener-8fa582fb6c7a21404f4927f1ac2befef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:53,504 INFO [StoreOpener-8fa582fb6c7a21404f4927f1ac2befef-1 {}] regionserver.HStore(327): Store=8fa582fb6c7a21404f4927f1ac2befef/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:57:53,504 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 8fa582fb6c7a21404f4927f1ac2befef 2024-11-25T03:57:53,505 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextException/8fa582fb6c7a21404f4927f1ac2befef 2024-11-25T03:57:53,506 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextException/8fa582fb6c7a21404f4927f1ac2befef 2024-11-25T03:57:53,507 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 8fa582fb6c7a21404f4927f1ac2befef 2024-11-25T03:57:53,507 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 8fa582fb6c7a21404f4927f1ac2befef 2024-11-25T03:57:53,510 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 8fa582fb6c7a21404f4927f1ac2befef 2024-11-25T03:57:53,513 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextException/8fa582fb6c7a21404f4927f1ac2befef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:57:53,514 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 8fa582fb6c7a21404f4927f1ac2befef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68634088, jitterRate=0.022727608680725098}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:57:53,514 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8fa582fb6c7a21404f4927f1ac2befef 2024-11-25T03:57:53,515 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 8fa582fb6c7a21404f4927f1ac2befef: Running coprocessor pre-open hook at 1732507073496Writing region info on filesystem at 1732507073496Initializing all the Stores at 1732507073498 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507073498Cleaning up temporary data from old regions at 1732507073507 (+9 ms)Running coprocessor post-open hooks at 1732507073514 (+7 ms)Region opened successfully at 1732507073515 (+1 ms) 2024-11-25T03:57:53,516 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef., pid=15, masterSystemTime=1732507073485 2024-11-25T03:57:53,519 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. 2024-11-25T03:57:53,519 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. 2024-11-25T03:57:53,520 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=8fa582fb6c7a21404f4927f1ac2befef, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:57:53,523 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fa582fb6c7a21404f4927f1ac2befef, server=d73269e94d72,33579,1732507016762 because future has completed 2024-11-25T03:57:53,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-25T03:57:53,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 8fa582fb6c7a21404f4927f1ac2befef, server=d73269e94d72,33579,1732507016762 in 193 msec 2024-11-25T03:57:53,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-25T03:57:53,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=8fa582fb6c7a21404f4927f1ac2befef, ASSIGN in 354 msec 2024-11-25T03:57:53,534 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:57:53,534 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507073534"}]},"ts":"1732507073534"} 2024-11-25T03:57:53,537 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-25T03:57:53,539 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:57:53,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2260 sec 2024-11-25T03:57:54,041 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-25T03:57:54,777 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:57:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:57:55,368 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-25T03:57:55,368 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-25T03:57:55,368 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:57:55,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-25T03:57:55,376 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:57:55,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-25T03:57:55,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef., hostname=d73269e94d72,33579,1732507016762, seqNum=2] 2024-11-25T03:57:55,394 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:57:55,394 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-25T03:57:55,398 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-25T03:57:55,401 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-25T03:57:55,402 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-25T03:57:55,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3839764233650700263 2024-11-25T03:57:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60454 deadline: 1732507078402, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:57:55,407 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-25T03:57:55,408 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-25T03:57:55,409 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:57:55,411 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:57:55,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3839764233650700261 2024-11-25T03:57:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60454 deadline: 1732507078411, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:57:55,416 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:57:55,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-3839764233650700260 2024-11-25T03:57:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60454 deadline: 1732507078415, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:57:55,420 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:57:55,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-3839764233650700259 2024-11-25T03:57:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60454 deadline: 1732507078419, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:57:55,424 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:57:55,433 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 225), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=151 (was 165), ProcessCount=11 (was 11), AvailableMemoryMB=9357 (was 9370) 2024-11-25T03:57:55,441 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=151, ProcessCount=11, AvailableMemoryMB=9356 2024-11-25T03:57:55,444 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:57:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-25T03:57:55,447 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:57:55,448 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:55,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-25T03:57:55,449 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:57:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-25T03:57:55,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741843_1019 (size=418) 2024-11-25T03:57:55,862 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1f0682440d0777c94f9662f5774a99e3, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc 2024-11-25T03:57:55,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741844_1020 (size=69) 2024-11-25T03:57:56,274 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:57:56,274 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 1f0682440d0777c94f9662f5774a99e3, disabling compactions & flushes 2024-11-25T03:57:56,274 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. 2024-11-25T03:57:56,274 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. 2024-11-25T03:57:56,275 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. after waiting 0 ms 2024-11-25T03:57:56,275 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. 2024-11-25T03:57:56,275 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. 2024-11-25T03:57:56,275 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1f0682440d0777c94f9662f5774a99e3: Waiting for close lock at 1732507076274Disabling compacts and flushes for region at 1732507076274Disabling writes for close at 1732507076275 (+1 ms)Writing region close event to WAL at 1732507076275Closed at 1732507076275 2024-11-25T03:57:56,278 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:57:56,279 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732507076278"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732507076278"}]},"ts":"1732507076278"} 2024-11-25T03:57:56,283 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:57:56,285 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:57:56,285 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507076285"}]},"ts":"1732507076285"} 2024-11-25T03:57:56,289 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-25T03:57:56,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=1f0682440d0777c94f9662f5774a99e3, ASSIGN}] 2024-11-25T03:57:56,291 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=1f0682440d0777c94f9662f5774a99e3, ASSIGN 2024-11-25T03:57:56,293 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=1f0682440d0777c94f9662f5774a99e3, ASSIGN; state=OFFLINE, location=d73269e94d72,33579,1732507016762; forceNewPlan=false, retain=false 2024-11-25T03:57:56,317 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-25T03:57:56,318 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-25T03:57:56,444 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=1f0682440d0777c94f9662f5774a99e3, regionState=OPENING, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:57:56,448 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=1f0682440d0777c94f9662f5774a99e3, ASSIGN because future has completed 2024-11-25T03:57:56,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f0682440d0777c94f9662f5774a99e3, server=d73269e94d72,33579,1732507016762}] 2024-11-25T03:57:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-25T03:57:56,608 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. 2024-11-25T03:57:56,608 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f0682440d0777c94f9662f5774a99e3, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:57:56,608 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 1f0682440d0777c94f9662f5774a99e3 2024-11-25T03:57:56,608 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:57:56,609 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 1f0682440d0777c94f9662f5774a99e3 2024-11-25T03:57:56,609 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 1f0682440d0777c94f9662f5774a99e3 2024-11-25T03:57:56,611 INFO [StoreOpener-1f0682440d0777c94f9662f5774a99e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1f0682440d0777c94f9662f5774a99e3 2024-11-25T03:57:56,613 INFO [StoreOpener-1f0682440d0777c94f9662f5774a99e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f0682440d0777c94f9662f5774a99e3 columnFamilyName testFamily 2024-11-25T03:57:56,613 DEBUG [StoreOpener-1f0682440d0777c94f9662f5774a99e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:57:56,614 INFO [StoreOpener-1f0682440d0777c94f9662f5774a99e3-1 {}] regionserver.HStore(327): Store=1f0682440d0777c94f9662f5774a99e3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:57:56,614 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 1f0682440d0777c94f9662f5774a99e3 2024-11-25T03:57:56,615 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScanner/1f0682440d0777c94f9662f5774a99e3 2024-11-25T03:57:56,616 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScanner/1f0682440d0777c94f9662f5774a99e3 2024-11-25T03:57:56,617 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 1f0682440d0777c94f9662f5774a99e3 2024-11-25T03:57:56,617 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 1f0682440d0777c94f9662f5774a99e3 2024-11-25T03:57:56,619 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 1f0682440d0777c94f9662f5774a99e3 2024-11-25T03:57:56,622 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScanner/1f0682440d0777c94f9662f5774a99e3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:57:56,623 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 1f0682440d0777c94f9662f5774a99e3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61676956, jitterRate=-0.08094173669815063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:57:56,623 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f0682440d0777c94f9662f5774a99e3 2024-11-25T03:57:56,624 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 1f0682440d0777c94f9662f5774a99e3: Running coprocessor pre-open hook at 1732507076609Writing region info on filesystem at 1732507076609Initializing all the Stores at 1732507076610 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507076610Cleaning up temporary data from old regions at 1732507076617 (+7 ms)Running coprocessor post-open hooks at 1732507076623 (+6 ms)Region opened successfully at 1732507076624 (+1 ms) 2024-11-25T03:57:56,625 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3., pid=18, masterSystemTime=1732507076601 2024-11-25T03:57:56,628 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. 2024-11-25T03:57:56,628 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. 2024-11-25T03:57:56,629 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=1f0682440d0777c94f9662f5774a99e3, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:57:56,631 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f0682440d0777c94f9662f5774a99e3, server=d73269e94d72,33579,1732507016762 because future has completed 2024-11-25T03:57:56,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-25T03:57:56,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 1f0682440d0777c94f9662f5774a99e3, server=d73269e94d72,33579,1732507016762 in 184 msec 2024-11-25T03:57:56,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-25T03:57:56,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=1f0682440d0777c94f9662f5774a99e3, ASSIGN in 346 msec 2024-11-25T03:57:56,639 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:57:56,639 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507076639"}]},"ts":"1732507076639"} 2024-11-25T03:57:56,642 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-25T03:57:56,643 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:57:56,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.1990 sec 2024-11-25T03:57:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-25T03:57:58,476 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-25T03:57:58,476 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-25T03:57:58,476 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:57:58,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-25T03:57:58,484 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:57:58,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-25T03:57:58,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3., hostname=d73269e94d72,33579,1732507016762, seqNum=2] 2024-11-25T03:57:58,498 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:57:58,498 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-25T03:57:58,501 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-25T03:57:58,503 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-25T03:57:58,998 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:57:59,511 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3., hostname=d73269e94d72,33579,1732507016762, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3., hostname=d73269e94d72,33579,1732507016762, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d73269e94d72:33579 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-25T03:57:59,512 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3., hostname=d73269e94d72,33579,1732507016762, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-25T03:57:59,512 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3., hostname=d73269e94d72,33579,1732507016762, seqNum=2 from cache 2024-11-25T03:58:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-25T03:58:00,043 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-25T03:58:00,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3., hostname=d73269e94d72,33579,1732507016762, seqNum=2] 2024-11-25T03:58:00,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-25T03:58:01,512 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3., hostname=d73269e94d72,33579,1732507016762, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3., hostname=d73269e94d72,33579,1732507016762, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d73269e94d72:33579 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=980ms, rpcTimeout=971ms 2024-11-25T03:58:01,512 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3., hostname=d73269e94d72,33579,1732507016762, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=980ms, rpcTimeout=971ms 2024-11-25T03:58:01,512 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3., hostname=d73269e94d72,33579,1732507016762, seqNum=2 from cache 2024-11-25T03:58:01,531 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=139 (was 151), ProcessCount=11 (was 11), AvailableMemoryMB=9277 (was 9356) 2024-11-25T03:58:01,540 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=139, ProcessCount=11, AvailableMemoryMB=9277 2024-11-25T03:58:01,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:58:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-25T03:58:01,587 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:58:01,588 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:58:01,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-25T03:58:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:58:01,591 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:58:01,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741845_1021 (size=416) 2024-11-25T03:58:02,009 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dac838663cb22cc4277bc6a8a58b14b5, NAME => 'testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc 2024-11-25T03:58:02,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741846_1022 (size=67) 2024-11-25T03:58:02,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-25T03:58:02,423 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:58:02,424 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing dac838663cb22cc4277bc6a8a58b14b5, disabling compactions & flushes 2024-11-25T03:58:02,424 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. 2024-11-25T03:58:02,424 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. 2024-11-25T03:58:02,424 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. after waiting 0 ms 2024-11-25T03:58:02,424 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. 2024-11-25T03:58:02,424 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. 2024-11-25T03:58:02,425 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for dac838663cb22cc4277bc6a8a58b14b5: Waiting for close lock at 1732507082424Disabling compacts and flushes for region at 1732507082424Disabling writes for close at 1732507082424Writing region close event to WAL at 1732507082424Closed at 1732507082424 2024-11-25T03:58:02,427 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:58:02,428 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732507082428"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732507082428"}]},"ts":"1732507082428"} 2024-11-25T03:58:02,432 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:58:02,433 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:58:02,434 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507082434"}]},"ts":"1732507082434"} 2024-11-25T03:58:02,437 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-25T03:58:02,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=dac838663cb22cc4277bc6a8a58b14b5, ASSIGN}] 2024-11-25T03:58:02,440 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=dac838663cb22cc4277bc6a8a58b14b5, ASSIGN 2024-11-25T03:58:02,442 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=dac838663cb22cc4277bc6a8a58b14b5, ASSIGN; state=OFFLINE, location=d73269e94d72,33579,1732507016762; forceNewPlan=false, retain=false 2024-11-25T03:58:02,593 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=dac838663cb22cc4277bc6a8a58b14b5, regionState=OPENING, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:58:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:58:02,600 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=dac838663cb22cc4277bc6a8a58b14b5, ASSIGN because future has completed 2024-11-25T03:58:02,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure dac838663cb22cc4277bc6a8a58b14b5, server=d73269e94d72,33579,1732507016762}] 2024-11-25T03:58:02,770 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. 2024-11-25T03:58:02,770 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => dac838663cb22cc4277bc6a8a58b14b5, NAME => 'testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:58:02,771 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync dac838663cb22cc4277bc6a8a58b14b5 2024-11-25T03:58:02,771 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:58:02,771 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for dac838663cb22cc4277bc6a8a58b14b5 2024-11-25T03:58:02,771 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for dac838663cb22cc4277bc6a8a58b14b5 2024-11-25T03:58:02,773 INFO [StoreOpener-dac838663cb22cc4277bc6a8a58b14b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region dac838663cb22cc4277bc6a8a58b14b5 2024-11-25T03:58:02,776 INFO [StoreOpener-dac838663cb22cc4277bc6a8a58b14b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dac838663cb22cc4277bc6a8a58b14b5 columnFamilyName testFamily 2024-11-25T03:58:02,776 DEBUG [StoreOpener-dac838663cb22cc4277bc6a8a58b14b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:58:02,777 INFO [StoreOpener-dac838663cb22cc4277bc6a8a58b14b5-1 {}] regionserver.HStore(327): Store=dac838663cb22cc4277bc6a8a58b14b5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:58:02,777 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for dac838663cb22cc4277bc6a8a58b14b5 2024-11-25T03:58:02,778 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNextAsync/dac838663cb22cc4277bc6a8a58b14b5 2024-11-25T03:58:02,779 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNextAsync/dac838663cb22cc4277bc6a8a58b14b5 2024-11-25T03:58:02,780 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for dac838663cb22cc4277bc6a8a58b14b5 2024-11-25T03:58:02,780 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for dac838663cb22cc4277bc6a8a58b14b5 2024-11-25T03:58:02,783 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for dac838663cb22cc4277bc6a8a58b14b5 2024-11-25T03:58:02,787 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNextAsync/dac838663cb22cc4277bc6a8a58b14b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:58:02,788 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened dac838663cb22cc4277bc6a8a58b14b5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75105164, jitterRate=0.11915415525436401}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:58:02,788 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dac838663cb22cc4277bc6a8a58b14b5 2024-11-25T03:58:02,789 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for dac838663cb22cc4277bc6a8a58b14b5: Running coprocessor pre-open hook at 1732507082771Writing region info on filesystem at 1732507082771Initializing all the Stores at 1732507082773 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507082773Cleaning up temporary data from old regions at 1732507082780 (+7 ms)Running coprocessor post-open hooks at 1732507082788 (+8 ms)Region opened successfully at 1732507082789 (+1 ms) 2024-11-25T03:58:02,790 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5., pid=21, masterSystemTime=1732507082756 2024-11-25T03:58:02,794 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. 2024-11-25T03:58:02,794 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. 2024-11-25T03:58:02,795 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=dac838663cb22cc4277bc6a8a58b14b5, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:58:02,798 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure dac838663cb22cc4277bc6a8a58b14b5, server=d73269e94d72,33579,1732507016762 because future has completed 2024-11-25T03:58:02,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-25T03:58:02,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure dac838663cb22cc4277bc6a8a58b14b5, server=d73269e94d72,33579,1732507016762 in 199 msec 2024-11-25T03:58:02,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-25T03:58:02,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=dac838663cb22cc4277bc6a8a58b14b5, ASSIGN in 365 msec 2024-11-25T03:58:02,807 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:58:02,807 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507082807"}]},"ts":"1732507082807"} 2024-11-25T03:58:02,810 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-25T03:58:02,811 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:58:02,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2680 sec 2024-11-25T03:58:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:58:04,627 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-25T03:58:04,627 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-25T03:58:04,628 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:58:04,640 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-25T03:58:04,640 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:58:04,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-25T03:58:04,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5., hostname=d73269e94d72,33579,1732507016762, seqNum=2] 2024-11-25T03:58:04,654 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:58:04,655 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-25T03:58:04,658 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-25T03:58:04,660 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-25T03:58:04,661 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-25T03:58:04,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-25T03:58:06,046 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-25T03:58:06,318 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-25T03:58:06,318 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-25T03:58:06,321 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-25T03:58:06,321 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T03:58:07,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3839764233650700248' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-25T03:58:07,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33579 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:60454 deadline: 1732507088674, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3839764233650700248' does not exist 2024-11-25T03:58:07,676 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to d73269e94d72,33579,1732507016762 for closing scanner id = -3839764233650700248 for dac838663cb22cc4277bc6a8a58b14b5 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3839764233650700248' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:58:07,682 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=128 (was 139), ProcessCount=11 (was 11), AvailableMemoryMB=9269 (was 9277) 2024-11-25T03:58:07,689 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=128, ProcessCount=11, AvailableMemoryMB=9269 2024-11-25T03:58:07,691 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:58:07,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-25T03:58:07,694 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:58:07,694 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:58:07,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-25T03:58:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-25T03:58:07,696 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:58:07,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741847_1023 (size=411) 2024-11-25T03:58:08,110 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 72ee6791c3678080578ac06256642fd0, NAME => 'testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc 2024-11-25T03:58:08,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741848_1024 (size=62) 2024-11-25T03:58:08,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-25T03:58:08,274 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:58:08,525 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:58:08,525 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 72ee6791c3678080578ac06256642fd0, disabling compactions & flushes 2024-11-25T03:58:08,526 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. 2024-11-25T03:58:08,526 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. 2024-11-25T03:58:08,526 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. after waiting 0 ms 2024-11-25T03:58:08,526 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. 2024-11-25T03:58:08,526 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. 2024-11-25T03:58:08,527 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 72ee6791c3678080578ac06256642fd0: Waiting for close lock at 1732507088525Disabling compacts and flushes for region at 1732507088525Disabling writes for close at 1732507088526 (+1 ms)Writing region close event to WAL at 1732507088526Closed at 1732507088526 2024-11-25T03:58:08,531 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:58:08,532 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732507088532"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732507088532"}]},"ts":"1732507088532"} 2024-11-25T03:58:08,536 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:58:08,538 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:58:08,538 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507088538"}]},"ts":"1732507088538"} 2024-11-25T03:58:08,541 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-25T03:58:08,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=72ee6791c3678080578ac06256642fd0, ASSIGN}] 2024-11-25T03:58:08,543 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=72ee6791c3678080578ac06256642fd0, ASSIGN 2024-11-25T03:58:08,545 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=72ee6791c3678080578ac06256642fd0, ASSIGN; state=OFFLINE, location=d73269e94d72,33579,1732507016762; forceNewPlan=false, retain=false 2024-11-25T03:58:08,696 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=72ee6791c3678080578ac06256642fd0, regionState=OPENING, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:58:08,704 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=72ee6791c3678080578ac06256642fd0, ASSIGN because future has completed 2024-11-25T03:58:08,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72ee6791c3678080578ac06256642fd0, server=d73269e94d72,33579,1732507016762}] 2024-11-25T03:58:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-25T03:58:08,872 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. 2024-11-25T03:58:08,872 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 72ee6791c3678080578ac06256642fd0, NAME => 'testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:58:08,873 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 72ee6791c3678080578ac06256642fd0 2024-11-25T03:58:08,873 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:58:08,873 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 72ee6791c3678080578ac06256642fd0 2024-11-25T03:58:08,873 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 72ee6791c3678080578ac06256642fd0 2024-11-25T03:58:08,875 INFO [StoreOpener-72ee6791c3678080578ac06256642fd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 72ee6791c3678080578ac06256642fd0 2024-11-25T03:58:08,877 INFO [StoreOpener-72ee6791c3678080578ac06256642fd0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72ee6791c3678080578ac06256642fd0 columnFamilyName testFamily 2024-11-25T03:58:08,877 DEBUG [StoreOpener-72ee6791c3678080578ac06256642fd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:58:08,878 INFO [StoreOpener-72ee6791c3678080578ac06256642fd0-1 {}] regionserver.HStore(327): Store=72ee6791c3678080578ac06256642fd0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:58:08,878 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 72ee6791c3678080578ac06256642fd0 2024-11-25T03:58:08,879 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNext/72ee6791c3678080578ac06256642fd0 2024-11-25T03:58:08,880 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNext/72ee6791c3678080578ac06256642fd0 2024-11-25T03:58:08,881 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 72ee6791c3678080578ac06256642fd0 2024-11-25T03:58:08,881 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 72ee6791c3678080578ac06256642fd0 2024-11-25T03:58:08,884 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 72ee6791c3678080578ac06256642fd0 2024-11-25T03:58:08,887 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNext/72ee6791c3678080578ac06256642fd0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:58:08,888 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 72ee6791c3678080578ac06256642fd0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72935913, jitterRate=0.08682979643344879}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:58:08,888 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 72ee6791c3678080578ac06256642fd0 2024-11-25T03:58:08,889 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 72ee6791c3678080578ac06256642fd0: Running coprocessor pre-open hook at 1732507088873Writing region info on filesystem at 1732507088873Initializing all the Stores at 1732507088875 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507088875Cleaning up temporary data from old regions at 1732507088881 (+6 ms)Running coprocessor post-open hooks at 1732507088888 (+7 ms)Region opened successfully at 1732507088888 2024-11-25T03:58:08,890 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0., pid=24, masterSystemTime=1732507088859 2024-11-25T03:58:08,892 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. 2024-11-25T03:58:08,892 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. 2024-11-25T03:58:08,893 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=72ee6791c3678080578ac06256642fd0, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:58:08,895 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72ee6791c3678080578ac06256642fd0, server=d73269e94d72,33579,1732507016762 because future has completed 2024-11-25T03:58:08,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-25T03:58:08,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 72ee6791c3678080578ac06256642fd0, server=d73269e94d72,33579,1732507016762 in 193 msec 2024-11-25T03:58:08,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-25T03:58:08,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=72ee6791c3678080578ac06256642fd0, ASSIGN in 358 msec 2024-11-25T03:58:08,903 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:58:08,904 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507088903"}]},"ts":"1732507088903"} 2024-11-25T03:58:08,906 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-25T03:58:08,907 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:58:08,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2160 sec 2024-11-25T03:58:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-25T03:58:10,726 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-25T03:58:10,726 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-25T03:58:10,727 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:58:10,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-25T03:58:10,737 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:58:10,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-25T03:58:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0., hostname=d73269e94d72,33579,1732507016762, seqNum=2] 2024-11-25T03:58:10,751 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:58:10,751 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-25T03:58:10,754 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-25T03:58:10,756 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-25T03:58:10,757 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-25T03:58:10,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-25T03:58:10,764 INFO [master/d73269e94d72:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-25T03:58:10,764 INFO [master/d73269e94d72:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-25T03:58:12,049 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-25T03:58:13,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3839764233650700244' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-25T03:58:13,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33579 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:60454 deadline: 1732507094772, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3839764233650700244' does not exist 2024-11-25T03:58:13,774 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to d73269e94d72,33579,1732507016762 for closing scanner id = -3839764233650700244 for 72ee6791c3678080578ac06256642fd0 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3839764233650700244' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:58:13,779 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=124 (was 128), ProcessCount=11 (was 11), AvailableMemoryMB=9264 (was 9269) 2024-11-25T03:58:13,787 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=124, ProcessCount=11, AvailableMemoryMB=9264 2024-11-25T03:58:13,789 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:58:13,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-25T03:58:13,792 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:58:13,792 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:58:13,792 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-25T03:58:13,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:58:13,794 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:58:13,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741849_1025 (size=419) 2024-11-25T03:58:14,210 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => be795f30588806ab35ea0484248ab990, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc 2024-11-25T03:58:14,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741850_1026 (size=70) 2024-11-25T03:58:14,222 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:58:14,223 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing be795f30588806ab35ea0484248ab990, disabling compactions & flushes 2024-11-25T03:58:14,223 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. 2024-11-25T03:58:14,223 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. 2024-11-25T03:58:14,223 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. after waiting 0 ms 2024-11-25T03:58:14,223 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. 2024-11-25T03:58:14,223 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. 2024-11-25T03:58:14,223 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for be795f30588806ab35ea0484248ab990: Waiting for close lock at 1732507094223Disabling compacts and flushes for region at 1732507094223Disabling writes for close at 1732507094223Writing region close event to WAL at 1732507094223Closed at 1732507094223 2024-11-25T03:58:14,224 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:58:14,225 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732507094224"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732507094224"}]},"ts":"1732507094224"} 2024-11-25T03:58:14,227 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:58:14,228 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:58:14,228 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507094228"}]},"ts":"1732507094228"} 2024-11-25T03:58:14,231 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-25T03:58:14,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=be795f30588806ab35ea0484248ab990, ASSIGN}] 2024-11-25T03:58:14,232 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=be795f30588806ab35ea0484248ab990, ASSIGN 2024-11-25T03:58:14,233 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=be795f30588806ab35ea0484248ab990, ASSIGN; state=OFFLINE, location=d73269e94d72,33579,1732507016762; forceNewPlan=false, retain=false 2024-11-25T03:58:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-25T03:58:14,376 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:58:14,384 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=be795f30588806ab35ea0484248ab990, regionState=OPENING, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:58:14,389 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=be795f30588806ab35ea0484248ab990, ASSIGN because future has completed 2024-11-25T03:58:14,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure be795f30588806ab35ea0484248ab990, server=d73269e94d72,33579,1732507016762}] 2024-11-25T03:58:14,555 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. 2024-11-25T03:58:14,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => be795f30588806ab35ea0484248ab990, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:58:14,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync be795f30588806ab35ea0484248ab990 2024-11-25T03:58:14,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:58:14,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for be795f30588806ab35ea0484248ab990 2024-11-25T03:58:14,556 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for be795f30588806ab35ea0484248ab990 2024-11-25T03:58:14,558 INFO [StoreOpener-be795f30588806ab35ea0484248ab990-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region be795f30588806ab35ea0484248ab990 2024-11-25T03:58:14,561 INFO [StoreOpener-be795f30588806ab35ea0484248ab990-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be795f30588806ab35ea0484248ab990 columnFamilyName testFamily 2024-11-25T03:58:14,561 DEBUG [StoreOpener-be795f30588806ab35ea0484248ab990-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:58:14,562 INFO [StoreOpener-be795f30588806ab35ea0484248ab990-1 {}] regionserver.HStore(327): Store=be795f30588806ab35ea0484248ab990/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:58:14,562 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for be795f30588806ab35ea0484248ab990 2024-11-25T03:58:14,563 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/be795f30588806ab35ea0484248ab990 2024-11-25T03:58:14,564 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/be795f30588806ab35ea0484248ab990 2024-11-25T03:58:14,565 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for be795f30588806ab35ea0484248ab990 2024-11-25T03:58:14,565 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for be795f30588806ab35ea0484248ab990 2024-11-25T03:58:14,567 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for be795f30588806ab35ea0484248ab990 2024-11-25T03:58:14,570 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/be795f30588806ab35ea0484248ab990/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:58:14,571 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened be795f30588806ab35ea0484248ab990; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72667720, jitterRate=0.08283340930938721}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:58:14,571 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be795f30588806ab35ea0484248ab990 2024-11-25T03:58:14,572 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for be795f30588806ab35ea0484248ab990: Running coprocessor pre-open hook at 1732507094556Writing region info on filesystem at 1732507094556Initializing all the Stores at 1732507094557 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507094557Cleaning up temporary data from old regions at 1732507094565 (+8 ms)Running coprocessor post-open hooks at 1732507094571 (+6 ms)Region opened successfully at 1732507094572 (+1 ms) 2024-11-25T03:58:14,573 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990., pid=27, masterSystemTime=1732507094542 2024-11-25T03:58:14,576 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. 2024-11-25T03:58:14,576 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. 2024-11-25T03:58:14,576 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=be795f30588806ab35ea0484248ab990, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:58:14,579 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure be795f30588806ab35ea0484248ab990, server=d73269e94d72,33579,1732507016762 because future has completed 2024-11-25T03:58:14,582 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-25T03:58:14,582 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure be795f30588806ab35ea0484248ab990, server=d73269e94d72,33579,1732507016762 in 191 msec 2024-11-25T03:58:14,584 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-25T03:58:14,584 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=be795f30588806ab35ea0484248ab990, ASSIGN in 351 msec 2024-11-25T03:58:14,585 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:58:14,585 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507094585"}]},"ts":"1732507094585"} 2024-11-25T03:58:14,587 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-25T03:58:14,589 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:58:14,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 800 msec 2024-11-25T03:58:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:58:14,806 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-25T03:58:14,806 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-25T03:58:14,807 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:58:14,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-25T03:58:14,818 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:58:14,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-25T03:58:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990., hostname=d73269e94d72,33579,1732507016762, seqNum=2] 2024-11-25T03:58:14,830 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:58:14,830 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-25T03:58:14,832 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-25T03:58:14,834 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-25T03:58:16,318 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-25T03:58:16,318 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-25T03:58:16,320 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-25T03:58:16,321 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T03:58:18,055 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-25T03:58:20,061 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:58:20,842 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990., hostname=d73269e94d72,33579,1732507016762, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990., hostname=d73269e94d72,33579,1732507016762, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d73269e94d72:33579 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-11-25T03:58:20,842 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990., hostname=d73269e94d72,33579,1732507016762, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-11-25T03:58:20,842 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990., hostname=d73269e94d72,33579,1732507016762, seqNum=2 from cache 2024-11-25T03:58:20,851 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=122 (was 124), ProcessCount=11 (was 11), AvailableMemoryMB=9254 (was 9264) 2024-11-25T03:58:20,858 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=122, ProcessCount=11, AvailableMemoryMB=9254 2024-11-25T03:58:20,860 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:58:20,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-25T03:58:20,863 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:58:20,863 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:58:20,863 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38513 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-25T03:58:20,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-25T03:58:20,865 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:58:20,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741851_1027 (size=407) 2024-11-25T03:58:21,277 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 28347e36b87c7acc164e40cb639068d6, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc 2024-11-25T03:58:21,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741852_1028 (size=58) 2024-11-25T03:58:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-25T03:58:21,688 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:58:21,688 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 28347e36b87c7acc164e40cb639068d6, disabling compactions & flushes 2024-11-25T03:58:21,688 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. 2024-11-25T03:58:21,688 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. 2024-11-25T03:58:21,688 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. after waiting 0 ms 2024-11-25T03:58:21,688 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. 2024-11-25T03:58:21,688 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. 2024-11-25T03:58:21,689 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 28347e36b87c7acc164e40cb639068d6: Waiting for close lock at 1732507101688Disabling compacts and flushes for region at 1732507101688Disabling writes for close at 1732507101688Writing region close event to WAL at 1732507101688Closed at 1732507101688 2024-11-25T03:58:21,692 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:58:21,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732507101692"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732507101692"}]},"ts":"1732507101692"} 2024-11-25T03:58:21,697 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:58:21,698 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:58:21,699 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507101698"}]},"ts":"1732507101698"} 2024-11-25T03:58:21,701 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-25T03:58:21,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=28347e36b87c7acc164e40cb639068d6, ASSIGN}] 2024-11-25T03:58:21,703 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=28347e36b87c7acc164e40cb639068d6, ASSIGN 2024-11-25T03:58:21,704 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=28347e36b87c7acc164e40cb639068d6, ASSIGN; state=OFFLINE, location=d73269e94d72,33579,1732507016762; forceNewPlan=false, retain=false 2024-11-25T03:58:21,856 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=28347e36b87c7acc164e40cb639068d6, regionState=OPENING, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:58:21,861 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=28347e36b87c7acc164e40cb639068d6, ASSIGN because future has completed 2024-11-25T03:58:21,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28347e36b87c7acc164e40cb639068d6, server=d73269e94d72,33579,1732507016762}] 2024-11-25T03:58:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-25T03:58:22,022 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. 2024-11-25T03:58:22,022 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 28347e36b87c7acc164e40cb639068d6, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:58:22,023 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 28347e36b87c7acc164e40cb639068d6 2024-11-25T03:58:22,023 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:58:22,023 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 28347e36b87c7acc164e40cb639068d6 2024-11-25T03:58:22,023 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 28347e36b87c7acc164e40cb639068d6 2024-11-25T03:58:22,026 INFO [StoreOpener-28347e36b87c7acc164e40cb639068d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 28347e36b87c7acc164e40cb639068d6 2024-11-25T03:58:22,029 INFO [StoreOpener-28347e36b87c7acc164e40cb639068d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28347e36b87c7acc164e40cb639068d6 columnFamilyName testFamily 2024-11-25T03:58:22,029 DEBUG [StoreOpener-28347e36b87c7acc164e40cb639068d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:58:22,030 INFO [StoreOpener-28347e36b87c7acc164e40cb639068d6-1 {}] regionserver.HStore(327): Store=28347e36b87c7acc164e40cb639068d6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:58:22,030 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 28347e36b87c7acc164e40cb639068d6 2024-11-25T03:58:22,031 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNext/28347e36b87c7acc164e40cb639068d6 2024-11-25T03:58:22,032 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNext/28347e36b87c7acc164e40cb639068d6 2024-11-25T03:58:22,033 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 28347e36b87c7acc164e40cb639068d6 2024-11-25T03:58:22,033 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 28347e36b87c7acc164e40cb639068d6 2024-11-25T03:58:22,036 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 28347e36b87c7acc164e40cb639068d6 2024-11-25T03:58:22,040 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNext/28347e36b87c7acc164e40cb639068d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:58:22,041 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 28347e36b87c7acc164e40cb639068d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61419063, jitterRate=-0.08478464186191559}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:58:22,042 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28347e36b87c7acc164e40cb639068d6 2024-11-25T03:58:22,042 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 28347e36b87c7acc164e40cb639068d6: Running coprocessor pre-open hook at 1732507102024Writing region info on filesystem at 1732507102024Initializing all the Stores at 1732507102026 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507102026Cleaning up temporary data from old regions at 1732507102033 (+7 ms)Running coprocessor post-open hooks at 1732507102042 (+9 ms)Region opened successfully at 1732507102042 2024-11-25T03:58:22,044 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6., pid=30, masterSystemTime=1732507102016 2024-11-25T03:58:22,047 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. 2024-11-25T03:58:22,047 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. 2024-11-25T03:58:22,048 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=28347e36b87c7acc164e40cb639068d6, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:58:22,051 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28347e36b87c7acc164e40cb639068d6, server=d73269e94d72,33579,1732507016762 because future has completed 2024-11-25T03:58:22,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-25T03:58:22,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 28347e36b87c7acc164e40cb639068d6, server=d73269e94d72,33579,1732507016762 in 190 msec 2024-11-25T03:58:22,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-25T03:58:22,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=28347e36b87c7acc164e40cb639068d6, ASSIGN in 353 msec 2024-11-25T03:58:22,058 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:58:22,058 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507102058"}]},"ts":"1732507102058"} 2024-11-25T03:58:22,060 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-25T03:58:22,061 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:58:22,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2010 sec 2024-11-25T03:58:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-25T03:58:23,887 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-25T03:58:23,887 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-25T03:58:23,888 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:58:23,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-25T03:58:23,900 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:58:23,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-25T03:58:23,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6., hostname=d73269e94d72,33579,1732507016762, seqNum=2] 2024-11-25T03:58:23,914 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:58:23,914 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-25T03:58:23,917 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-25T03:58:23,919 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-25T03:58:23,920 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-25T03:58:23,921 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-25T03:58:24,059 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-25T03:58:24,778 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:58:26,318 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-25T03:58:26,318 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-25T03:58:27,529 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:58:29,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33579 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3839764233650700236' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-25T03:58:29,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33579 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:60454 deadline: 1732507115922, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3839764233650700236' does not exist 2024-11-25T03:58:29,926 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to d73269e94d72,33579,1732507016762 for closing scanner id = -3839764233650700236 for 28347e36b87c7acc164e40cb639068d6 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3839764233650700236' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:58:29,930 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=103 (was 122), ProcessCount=11 (was 11), AvailableMemoryMB=9243 (was 9254) 2024-11-25T03:58:29,938 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=103, ProcessCount=11, AvailableMemoryMB=9242 2024-11-25T03:58:29,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:58:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-25T03:58:29,944 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:58:29,944 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:58:29,944 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-25T03:58:29,945 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:58:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:58:29,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741853_1029 (size=423) 2024-11-25T03:58:30,361 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 74fcbd80d22284921c26f33def236beb, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc 2024-11-25T03:58:30,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741854_1030 (size=74) 2024-11-25T03:58:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-25T03:58:30,726 DEBUG [master/d73269e94d72:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:58:30,772 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:58:30,772 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 74fcbd80d22284921c26f33def236beb, disabling compactions & flushes 2024-11-25T03:58:30,772 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. 2024-11-25T03:58:30,772 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. 2024-11-25T03:58:30,773 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. after waiting 0 ms 2024-11-25T03:58:30,773 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. 2024-11-25T03:58:30,773 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. 2024-11-25T03:58:30,773 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 74fcbd80d22284921c26f33def236beb: Waiting for close lock at 1732507110772Disabling compacts and flushes for region at 1732507110772Disabling writes for close at 1732507110773 (+1 ms)Writing region close event to WAL at 1732507110773Closed at 1732507110773 2024-11-25T03:58:30,775 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:58:30,775 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732507110775"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732507110775"}]},"ts":"1732507110775"} 2024-11-25T03:58:30,778 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:58:30,779 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:58:30,780 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507110779"}]},"ts":"1732507110779"} 2024-11-25T03:58:30,782 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-25T03:58:30,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=74fcbd80d22284921c26f33def236beb, ASSIGN}] 2024-11-25T03:58:30,785 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=74fcbd80d22284921c26f33def236beb, ASSIGN 2024-11-25T03:58:30,786 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=74fcbd80d22284921c26f33def236beb, ASSIGN; state=OFFLINE, location=d73269e94d72,33579,1732507016762; forceNewPlan=false, retain=false 2024-11-25T03:58:30,937 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=74fcbd80d22284921c26f33def236beb, regionState=OPENING, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:58:30,945 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=74fcbd80d22284921c26f33def236beb, ASSIGN because future has completed 2024-11-25T03:58:30,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74fcbd80d22284921c26f33def236beb, server=d73269e94d72,33579,1732507016762}] 2024-11-25T03:58:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:58:31,109 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. 2024-11-25T03:58:31,109 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 74fcbd80d22284921c26f33def236beb, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:58:31,110 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 74fcbd80d22284921c26f33def236beb 2024-11-25T03:58:31,110 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:58:31,110 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 74fcbd80d22284921c26f33def236beb 2024-11-25T03:58:31,110 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 74fcbd80d22284921c26f33def236beb 2024-11-25T03:58:31,112 INFO [StoreOpener-74fcbd80d22284921c26f33def236beb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 74fcbd80d22284921c26f33def236beb 2024-11-25T03:58:31,114 INFO [StoreOpener-74fcbd80d22284921c26f33def236beb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74fcbd80d22284921c26f33def236beb columnFamilyName testFamily 2024-11-25T03:58:31,114 DEBUG [StoreOpener-74fcbd80d22284921c26f33def236beb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:58:31,115 INFO [StoreOpener-74fcbd80d22284921c26f33def236beb-1 {}] regionserver.HStore(327): Store=74fcbd80d22284921c26f33def236beb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:58:31,115 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 74fcbd80d22284921c26f33def236beb 2024-11-25T03:58:31,116 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScannerAsync/74fcbd80d22284921c26f33def236beb 2024-11-25T03:58:31,117 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScannerAsync/74fcbd80d22284921c26f33def236beb 2024-11-25T03:58:31,118 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 74fcbd80d22284921c26f33def236beb 2024-11-25T03:58:31,118 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 74fcbd80d22284921c26f33def236beb 2024-11-25T03:58:31,120 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 74fcbd80d22284921c26f33def236beb 2024-11-25T03:58:31,123 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScannerAsync/74fcbd80d22284921c26f33def236beb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:58:31,124 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 74fcbd80d22284921c26f33def236beb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72544351, jitterRate=0.08099506795406342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:58:31,124 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 74fcbd80d22284921c26f33def236beb 2024-11-25T03:58:31,125 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 74fcbd80d22284921c26f33def236beb: Running coprocessor pre-open hook at 1732507111110Writing region info on filesystem at 1732507111110Initializing all the Stores at 1732507111112 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732507111112Cleaning up temporary data from old regions at 1732507111118 (+6 ms)Running coprocessor post-open hooks at 1732507111124 (+6 ms)Region opened successfully at 1732507111125 (+1 ms) 2024-11-25T03:58:31,126 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb., pid=33, masterSystemTime=1732507111099 2024-11-25T03:58:31,129 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. 2024-11-25T03:58:31,129 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. 2024-11-25T03:58:31,129 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=74fcbd80d22284921c26f33def236beb, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33579,1732507016762 2024-11-25T03:58:31,132 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74fcbd80d22284921c26f33def236beb, server=d73269e94d72,33579,1732507016762 because future has completed 2024-11-25T03:58:31,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-25T03:58:31,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 74fcbd80d22284921c26f33def236beb, server=d73269e94d72,33579,1732507016762 in 188 msec 2024-11-25T03:58:31,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-25T03:58:31,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=74fcbd80d22284921c26f33def236beb, ASSIGN in 353 msec 2024-11-25T03:58:31,139 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:58:31,139 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732507111139"}]},"ts":"1732507111139"} 2024-11-25T03:58:31,142 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-25T03:58:31,143 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:58:31,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2020 sec 2024-11-25T03:58:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38513 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:58:32,986 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-25T03:58:32,986 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-25T03:58:32,987 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:58:32,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-25T03:58:32,997 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:58:32,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-25T03:58:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb., hostname=d73269e94d72,33579,1732507016762, seqNum=2] 2024-11-25T03:58:33,009 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:58:33,009 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-25T03:58:33,012 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-25T03:58:33,014 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-25T03:58:34,022 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb., hostname=d73269e94d72,33579,1732507016762, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb., hostname=d73269e94d72,33579,1732507016762, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d73269e94d72:33579 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-25T03:58:34,023 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb., hostname=d73269e94d72,33579,1732507016762, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-25T03:58:34,023 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb., hostname=d73269e94d72,33579,1732507016762, seqNum=2 from cache 2024-11-25T03:58:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-25T03:58:35,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb., hostname=d73269e94d72,33579,1732507016762, seqNum=2] 2024-11-25T03:58:35,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33579 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-25T03:58:36,022 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb., hostname=d73269e94d72,33579,1732507016762, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb., hostname=d73269e94d72,33579,1732507016762, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d73269e94d72:33579 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=979ms, rpcTimeout=970ms 2024-11-25T03:58:36,022 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb., hostname=d73269e94d72,33579,1732507016762, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=979ms, rpcTimeout=970ms 2024-11-25T03:58:36,022 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb., hostname=d73269e94d72,33579,1732507016762, seqNum=2 from cache 2024-11-25T03:58:36,037 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=94 (was 103), ProcessCount=11 (was 11), AvailableMemoryMB=9253 (was 9242) - AvailableMemoryMB LEAK? - 2024-11-25T03:58:36,038 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:58:36,038 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:58:36,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:58:36,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:58:36,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:58:36,043 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T03:58:36,043 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:58:36,043 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:58:36,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:58:36,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:58:36,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:58:36,043 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T03:58:36,044 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1846701803, stopped=false 2024-11-25T03:58:36,044 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d73269e94d72,38513,1732507015876 2024-11-25T03:58:36,063 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-25T03:58:36,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:58:36,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:58:36,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:58:36,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:58:36,081 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:58:36,082 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:58:36,082 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:58:36,082 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:58:36,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:58:36,083 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd73269e94d72,33579,1732507016762' ***** 2024-11-25T03:58:36,083 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T03:58:36,083 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:58:36,084 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:58:36,084 INFO [RS:0;d73269e94d72:33579 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T03:58:36,084 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:58:36,085 INFO [RS:0;d73269e94d72:33579 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T03:58:36,085 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(3091): Received CLOSE for 1f0682440d0777c94f9662f5774a99e3 2024-11-25T03:58:36,086 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(3091): Received CLOSE for be795f30588806ab35ea0484248ab990 2024-11-25T03:58:36,086 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(3091): Received CLOSE for 28347e36b87c7acc164e40cb639068d6 2024-11-25T03:58:36,086 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(3091): Received CLOSE for dac838663cb22cc4277bc6a8a58b14b5 2024-11-25T03:58:36,086 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(3091): Received CLOSE for 7cc8c925e00abed8a7bf0e5438157cb7 2024-11-25T03:58:36,087 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(3091): Received CLOSE for 74fcbd80d22284921c26f33def236beb 2024-11-25T03:58:36,087 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(3091): Received CLOSE for 72ee6791c3678080578ac06256642fd0 2024-11-25T03:58:36,087 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1f0682440d0777c94f9662f5774a99e3, disabling compactions & flushes 2024-11-25T03:58:36,087 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(3091): Received CLOSE for 2d3b244c1794e17d8de0df895f56030d 2024-11-25T03:58:36,087 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. 2024-11-25T03:58:36,087 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(3091): Received CLOSE for fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:36,087 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. 2024-11-25T03:58:36,087 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(3091): Received CLOSE for 8fa582fb6c7a21404f4927f1ac2befef 2024-11-25T03:58:36,087 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. after waiting 0 ms 2024-11-25T03:58:36,087 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. 2024-11-25T03:58:36,087 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(959): stopping server d73269e94d72,33579,1732507016762 2024-11-25T03:58:36,088 INFO [RS:0;d73269e94d72:33579 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:58:36,088 INFO [RS:0;d73269e94d72:33579 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d73269e94d72:33579. 2024-11-25T03:58:36,088 DEBUG [RS:0;d73269e94d72:33579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:58:36,088 DEBUG [RS:0;d73269e94d72:33579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:58:36,088 INFO [RS:0;d73269e94d72:33579 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T03:58:36,089 INFO [RS:0;d73269e94d72:33579 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T03:58:36,089 INFO [RS:0;d73269e94d72:33579 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T03:58:36,089 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T03:58:36,089 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-25T03:58:36,089 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:58:36,089 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:58:36,090 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:58:36,090 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:58:36,090 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:58:36,090 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1f0682440d0777c94f9662f5774a99e3, 28347e36b87c7acc164e40cb639068d6, 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 74fcbd80d22284921c26f33def236beb, 7cc8c925e00abed8a7bf0e5438157cb7, 8fa582fb6c7a21404f4927f1ac2befef, be795f30588806ab35ea0484248ab990, dac838663cb22cc4277bc6a8a58b14b5, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:36,093 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1f0682440d0777c94f9662f5774a99e3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:58:36,093 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-25T03:58:36,144 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScanner/1f0682440d0777c94f9662f5774a99e3/.tmp/testFamily/3c823afccdc94beba6da5b92912e915c is 61, key is row-0/testFamily:testQualifier/1732507078488/Put/seqid=0 2024-11-25T03:58:36,149 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/.tmp/info/c103597d864046a7b2a87f69674bb1aa is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7./info:regioninfo/1732507052163/Put/seqid=0 2024-11-25T03:58:36,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741855_1031 (size=20564) 2024-11-25T03:58:36,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741856_1032 (size=5049) 2024-11-25T03:58:36,290 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1f0682440d0777c94f9662f5774a99e3, 28347e36b87c7acc164e40cb639068d6, 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 74fcbd80d22284921c26f33def236beb, 7cc8c925e00abed8a7bf0e5438157cb7, 8fa582fb6c7a21404f4927f1ac2befef, be795f30588806ab35ea0484248ab990, dac838663cb22cc4277bc6a8a58b14b5, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:36,491 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1f0682440d0777c94f9662f5774a99e3, 28347e36b87c7acc164e40cb639068d6, 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 74fcbd80d22284921c26f33def236beb, 7cc8c925e00abed8a7bf0e5438157cb7, 8fa582fb6c7a21404f4927f1ac2befef, be795f30588806ab35ea0484248ab990, dac838663cb22cc4277bc6a8a58b14b5, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:36,545 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-25T03:58:36,559 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScanner/1f0682440d0777c94f9662f5774a99e3/.tmp/testFamily/3c823afccdc94beba6da5b92912e915c 2024-11-25T03:58:36,559 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/.tmp/info/c103597d864046a7b2a87f69674bb1aa 2024-11-25T03:58:36,604 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScanner/1f0682440d0777c94f9662f5774a99e3/.tmp/testFamily/3c823afccdc94beba6da5b92912e915c as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScanner/1f0682440d0777c94f9662f5774a99e3/testFamily/3c823afccdc94beba6da5b92912e915c 2024-11-25T03:58:36,611 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:58:36,613 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScanner/1f0682440d0777c94f9662f5774a99e3/testFamily/3c823afccdc94beba6da5b92912e915c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:58:36,621 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1f0682440d0777c94f9662f5774a99e3 in 529ms, sequenceid=8, compaction requested=false 2024-11-25T03:58:36,626 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/.tmp/ns/bf90806859e0490a9d1b1171e2c591dc is 43, key is default/ns:d/1732507050670/Put/seqid=0 2024-11-25T03:58:36,627 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScanner/1f0682440d0777c94f9662f5774a99e3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:58:36,629 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. 2024-11-25T03:58:36,630 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1f0682440d0777c94f9662f5774a99e3: Waiting for close lock at 1732507116086Running coprocessor pre-close hooks at 1732507116087 (+1 ms)Disabling compacts and flushes for region at 1732507116087Disabling writes for close at 1732507116087Obtaining lock to block concurrent updates at 1732507116093 (+6 ms)Preparing flush snapshotting stores in 1f0682440d0777c94f9662f5774a99e3 at 1732507116093Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732507116099 (+6 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. at 1732507116101 (+2 ms)Flushing 1f0682440d0777c94f9662f5774a99e3/testFamily: creating writer at 1732507116102 (+1 ms)Flushing 1f0682440d0777c94f9662f5774a99e3/testFamily: appending metadata at 1732507116139 (+37 ms)Flushing 1f0682440d0777c94f9662f5774a99e3/testFamily: closing flushed file at 1732507116141 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35ed844f: reopening flushed file at 1732507116602 (+461 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1f0682440d0777c94f9662f5774a99e3 in 529ms, sequenceid=8, compaction requested=false at 1732507116621 (+19 ms)Writing region close event to WAL at 1732507116622 (+1 ms)Running coprocessor post-close hooks at 1732507116628 (+6 ms)Closed at 1732507116629 (+1 ms) 2024-11-25T03:58:36,630 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732507075443.1f0682440d0777c94f9662f5774a99e3. 2024-11-25T03:58:36,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741857_1033 (size=5153) 2024-11-25T03:58:36,631 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing be795f30588806ab35ea0484248ab990, disabling compactions & flushes 2024-11-25T03:58:36,631 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. 2024-11-25T03:58:36,631 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. 2024-11-25T03:58:36,631 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. after waiting 0 ms 2024-11-25T03:58:36,631 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. 2024-11-25T03:58:36,631 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing be795f30588806ab35ea0484248ab990 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:58:36,650 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-25T03:58:36,650 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T03:58:36,651 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/be795f30588806ab35ea0484248ab990/.tmp/testFamily/d72a2bc0912443ec9abb1c47916af45d is 61, key is row-0/testFamily:testQualifier/1732507094821/Put/seqid=0 2024-11-25T03:58:36,651 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-25T03:58:36,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741858_1034 (size=5049) 2024-11-25T03:58:36,691 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 28347e36b87c7acc164e40cb639068d6, 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 74fcbd80d22284921c26f33def236beb, 7cc8c925e00abed8a7bf0e5438157cb7, 8fa582fb6c7a21404f4927f1ac2befef, be795f30588806ab35ea0484248ab990, dac838663cb22cc4277bc6a8a58b14b5, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:36,892 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 28347e36b87c7acc164e40cb639068d6, 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 74fcbd80d22284921c26f33def236beb, 7cc8c925e00abed8a7bf0e5438157cb7, 8fa582fb6c7a21404f4927f1ac2befef, be795f30588806ab35ea0484248ab990, dac838663cb22cc4277bc6a8a58b14b5, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:36,986 INFO [regionserver/d73269e94d72:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-25T03:58:36,987 INFO [regionserver/d73269e94d72:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-25T03:58:37,032 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/.tmp/ns/bf90806859e0490a9d1b1171e2c591dc 2024-11-25T03:58:37,055 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/be795f30588806ab35ea0484248ab990/.tmp/testFamily/d72a2bc0912443ec9abb1c47916af45d 2024-11-25T03:58:37,064 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/be795f30588806ab35ea0484248ab990/.tmp/testFamily/d72a2bc0912443ec9abb1c47916af45d as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/be795f30588806ab35ea0484248ab990/testFamily/d72a2bc0912443ec9abb1c47916af45d 2024-11-25T03:58:37,068 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/.tmp/table/b514b32b28914f3fbadde087adf13141 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732507052181/Put/seqid=0 2024-11-25T03:58:37,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741859_1035 (size=6699) 2024-11-25T03:58:37,074 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/be795f30588806ab35ea0484248ab990/testFamily/d72a2bc0912443ec9abb1c47916af45d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:58:37,075 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for be795f30588806ab35ea0484248ab990 in 444ms, sequenceid=8, compaction requested=false 2024-11-25T03:58:37,080 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/be795f30588806ab35ea0484248ab990/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:58:37,081 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. 2024-11-25T03:58:37,081 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for be795f30588806ab35ea0484248ab990: Waiting for close lock at 1732507116631Running coprocessor pre-close hooks at 1732507116631Disabling compacts and flushes for region at 1732507116631Disabling writes for close at 1732507116631Obtaining lock to block concurrent updates at 1732507116631Preparing flush snapshotting stores in be795f30588806ab35ea0484248ab990 at 1732507116631Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732507116631Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. at 1732507116632 (+1 ms)Flushing be795f30588806ab35ea0484248ab990/testFamily: creating writer at 1732507116632Flushing be795f30588806ab35ea0484248ab990/testFamily: appending metadata at 1732507116650 (+18 ms)Flushing be795f30588806ab35ea0484248ab990/testFamily: closing flushed file at 1732507116650Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21d040d0: reopening flushed file at 1732507117063 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for be795f30588806ab35ea0484248ab990 in 444ms, sequenceid=8, compaction requested=false at 1732507117075 (+12 ms)Writing region close event to WAL at 1732507117076 (+1 ms)Running coprocessor post-close hooks at 1732507117081 (+5 ms)Closed at 1732507117081 2024-11-25T03:58:37,081 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732507093788.be795f30588806ab35ea0484248ab990. 2024-11-25T03:58:37,081 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 28347e36b87c7acc164e40cb639068d6, disabling compactions & flushes 2024-11-25T03:58:37,081 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. 2024-11-25T03:58:37,081 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. 2024-11-25T03:58:37,081 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. after waiting 0 ms 2024-11-25T03:58:37,082 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. 2024-11-25T03:58:37,082 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 28347e36b87c7acc164e40cb639068d6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:58:37,092 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-25T03:58:37,092 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1325): Online Regions={28347e36b87c7acc164e40cb639068d6=hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6., dac838663cb22cc4277bc6a8a58b14b5=testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5., 7cc8c925e00abed8a7bf0e5438157cb7=testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7., 74fcbd80d22284921c26f33def236beb=testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb., 1588230740=hbase:meta,,1.1588230740, 72ee6791c3678080578ac06256642fd0=testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0., 2d3b244c1794e17d8de0df895f56030d=hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d., fee51fafcdd578c2ae0ea3c13d1a4565=hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565., 8fa582fb6c7a21404f4927f1ac2befef=testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef.} 2024-11-25T03:58:37,092 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 28347e36b87c7acc164e40cb639068d6, 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 74fcbd80d22284921c26f33def236beb, 7cc8c925e00abed8a7bf0e5438157cb7, 8fa582fb6c7a21404f4927f1ac2befef, dac838663cb22cc4277bc6a8a58b14b5, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:37,098 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNext/28347e36b87c7acc164e40cb639068d6/.tmp/testFamily/0c75575e27734042a95efcea3deffd07 is 61, key is row-0/testFamily:testQualifier/1732507103905/Put/seqid=0 2024-11-25T03:58:37,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741860_1036 (size=5049) 2024-11-25T03:58:37,293 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 28347e36b87c7acc164e40cb639068d6, 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 74fcbd80d22284921c26f33def236beb, 7cc8c925e00abed8a7bf0e5438157cb7, 8fa582fb6c7a21404f4927f1ac2befef, dac838663cb22cc4277bc6a8a58b14b5, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:37,475 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/.tmp/table/b514b32b28914f3fbadde087adf13141 2024-11-25T03:58:37,490 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/.tmp/info/c103597d864046a7b2a87f69674bb1aa as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/info/c103597d864046a7b2a87f69674bb1aa 2024-11-25T03:58:37,493 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 28347e36b87c7acc164e40cb639068d6, 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 74fcbd80d22284921c26f33def236beb, 7cc8c925e00abed8a7bf0e5438157cb7, 8fa582fb6c7a21404f4927f1ac2befef, dac838663cb22cc4277bc6a8a58b14b5, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:37,497 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/info/c103597d864046a7b2a87f69674bb1aa, entries=100, sequenceid=56, filesize=20.1 K 2024-11-25T03:58:37,499 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/.tmp/ns/bf90806859e0490a9d1b1171e2c591dc as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/ns/bf90806859e0490a9d1b1171e2c591dc 2024-11-25T03:58:37,503 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNext/28347e36b87c7acc164e40cb639068d6/.tmp/testFamily/0c75575e27734042a95efcea3deffd07 2024-11-25T03:58:37,508 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/ns/bf90806859e0490a9d1b1171e2c591dc, entries=2, sequenceid=56, filesize=5.0 K 2024-11-25T03:58:37,509 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/.tmp/table/b514b32b28914f3fbadde087adf13141 as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/table/b514b32b28914f3fbadde087adf13141 2024-11-25T03:58:37,511 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNext/28347e36b87c7acc164e40cb639068d6/.tmp/testFamily/0c75575e27734042a95efcea3deffd07 as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNext/28347e36b87c7acc164e40cb639068d6/testFamily/0c75575e27734042a95efcea3deffd07 2024-11-25T03:58:37,517 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/table/b514b32b28914f3fbadde087adf13141, entries=20, sequenceid=56, filesize=6.5 K 2024-11-25T03:58:37,518 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1428ms, sequenceid=56, compaction requested=false 2024-11-25T03:58:37,518 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNext/28347e36b87c7acc164e40cb639068d6/testFamily/0c75575e27734042a95efcea3deffd07, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:58:37,519 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 28347e36b87c7acc164e40cb639068d6 in 437ms, sequenceid=8, compaction requested=false 2024-11-25T03:58:37,523 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-25T03:58:37,523 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNext/28347e36b87c7acc164e40cb639068d6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:58:37,524 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:58:37,524 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:58:37,524 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732507116089Running coprocessor pre-close hooks at 1732507116089Disabling compacts and flushes for region at 1732507116089Disabling writes for close at 1732507116090 (+1 ms)Obtaining lock to block concurrent updates at 1732507116093 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1732507116093Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732507116099 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732507116101 (+2 ms)Flushing 1588230740/info: creating writer at 1732507116102 (+1 ms)Flushing 1588230740/info: appending metadata at 1732507116144 (+42 ms)Flushing 1588230740/info: closing flushed file at 1732507116145 (+1 ms)Flushing 1588230740/ns: creating writer at 1732507116602 (+457 ms)Flushing 1588230740/ns: appending metadata at 1732507116624 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1732507116624Flushing 1588230740/table: creating writer at 1732507117047 (+423 ms)Flushing 1588230740/table: appending metadata at 1732507117067 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732507117067Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fad459b: reopening flushed file at 1732507117488 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bff1ef9: reopening flushed file at 1732507117498 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58219def: reopening flushed file at 1732507117508 (+10 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1428ms, sequenceid=56, compaction requested=false at 1732507117518 (+10 ms)Writing region close event to WAL at 1732507117519 (+1 ms)Running coprocessor post-close hooks at 1732507117524 (+5 ms)Closed at 1732507117524 2024-11-25T03:58:37,524 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. 2024-11-25T03:58:37,524 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 28347e36b87c7acc164e40cb639068d6: Waiting for close lock at 1732507117081Running coprocessor pre-close hooks at 1732507117081Disabling compacts and flushes for region at 1732507117081Disabling writes for close at 1732507117082 (+1 ms)Obtaining lock to block concurrent updates at 1732507117082Preparing flush snapshotting stores in 28347e36b87c7acc164e40cb639068d6 at 1732507117082Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732507117082Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. at 1732507117083 (+1 ms)Flushing 28347e36b87c7acc164e40cb639068d6/testFamily: creating writer at 1732507117083Flushing 28347e36b87c7acc164e40cb639068d6/testFamily: appending metadata at 1732507117097 (+14 ms)Flushing 28347e36b87c7acc164e40cb639068d6/testFamily: closing flushed file at 1732507117097Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1515c421: reopening flushed file at 1732507117510 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 28347e36b87c7acc164e40cb639068d6 in 437ms, sequenceid=8, compaction requested=false at 1732507117519 (+9 ms)Writing region close event to WAL at 1732507117520 (+1 ms)Running coprocessor post-close hooks at 1732507117524 (+4 ms)Closed at 1732507117524 2024-11-25T03:58:37,524 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T03:58:37,524 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732507100860.28347e36b87c7acc164e40cb639068d6. 2024-11-25T03:58:37,525 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dac838663cb22cc4277bc6a8a58b14b5, disabling compactions & flushes 2024-11-25T03:58:37,525 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. 2024-11-25T03:58:37,525 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. 2024-11-25T03:58:37,525 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. after waiting 0 ms 2024-11-25T03:58:37,525 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. 2024-11-25T03:58:37,525 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing dac838663cb22cc4277bc6a8a58b14b5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:58:37,540 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNextAsync/dac838663cb22cc4277bc6a8a58b14b5/.tmp/testFamily/3e481df5b19242a4adea7d3b8127aea9 is 61, key is row-0/testFamily:testQualifier/1732507084646/Put/seqid=0 2024-11-25T03:58:37,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741861_1037 (size=5049) 2024-11-25T03:58:37,693 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 74fcbd80d22284921c26f33def236beb, 7cc8c925e00abed8a7bf0e5438157cb7, 8fa582fb6c7a21404f4927f1ac2befef, dac838663cb22cc4277bc6a8a58b14b5, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:37,894 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 74fcbd80d22284921c26f33def236beb, 7cc8c925e00abed8a7bf0e5438157cb7, 8fa582fb6c7a21404f4927f1ac2befef, dac838663cb22cc4277bc6a8a58b14b5, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:37,947 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNextAsync/dac838663cb22cc4277bc6a8a58b14b5/.tmp/testFamily/3e481df5b19242a4adea7d3b8127aea9 2024-11-25T03:58:37,961 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNextAsync/dac838663cb22cc4277bc6a8a58b14b5/.tmp/testFamily/3e481df5b19242a4adea7d3b8127aea9 as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNextAsync/dac838663cb22cc4277bc6a8a58b14b5/testFamily/3e481df5b19242a4adea7d3b8127aea9 2024-11-25T03:58:37,969 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNextAsync/dac838663cb22cc4277bc6a8a58b14b5/testFamily/3e481df5b19242a4adea7d3b8127aea9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:58:37,970 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for dac838663cb22cc4277bc6a8a58b14b5 in 445ms, sequenceid=8, compaction requested=false 2024-11-25T03:58:37,975 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNextAsync/dac838663cb22cc4277bc6a8a58b14b5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:58:37,976 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. 2024-11-25T03:58:37,976 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dac838663cb22cc4277bc6a8a58b14b5: Waiting for close lock at 1732507117525Running coprocessor pre-close hooks at 1732507117525Disabling compacts and flushes for region at 1732507117525Disabling writes for close at 1732507117525Obtaining lock to block concurrent updates at 1732507117525Preparing flush snapshotting stores in dac838663cb22cc4277bc6a8a58b14b5 at 1732507117525Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732507117525Flushing stores of testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. at 1732507117526 (+1 ms)Flushing dac838663cb22cc4277bc6a8a58b14b5/testFamily: creating writer at 1732507117526Flushing dac838663cb22cc4277bc6a8a58b14b5/testFamily: appending metadata at 1732507117540 (+14 ms)Flushing dac838663cb22cc4277bc6a8a58b14b5/testFamily: closing flushed file at 1732507117540Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7631e33e: reopening flushed file at 1732507117960 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for dac838663cb22cc4277bc6a8a58b14b5 in 445ms, sequenceid=8, compaction requested=false at 1732507117970 (+10 ms)Writing region close event to WAL at 1732507117971 (+1 ms)Running coprocessor post-close hooks at 1732507117976 (+5 ms)Closed at 1732507117976 2024-11-25T03:58:37,976 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732507081542.dac838663cb22cc4277bc6a8a58b14b5. 2024-11-25T03:58:37,976 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7cc8c925e00abed8a7bf0e5438157cb7, disabling compactions & flushes 2024-11-25T03:58:37,976 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. 2024-11-25T03:58:37,976 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. 2024-11-25T03:58:37,976 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. after waiting 0 ms 2024-11-25T03:58:37,976 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. 2024-11-25T03:58:37,976 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7cc8c925e00abed8a7bf0e5438157cb7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:58:37,992 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7cc8c925e00abed8a7bf0e5438157cb7/.tmp/testFamily/745ea443a1934f50a4ac4ad8725b4cfb is 61, key is row-0/testFamily:testQualifier/1732507054024/Put/seqid=0 2024-11-25T03:58:37,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741862_1038 (size=5049) 2024-11-25T03:58:38,094 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-25T03:58:38,094 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1325): Online Regions={7cc8c925e00abed8a7bf0e5438157cb7=testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7., 74fcbd80d22284921c26f33def236beb=testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb., 72ee6791c3678080578ac06256642fd0=testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0., 2d3b244c1794e17d8de0df895f56030d=hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d., fee51fafcdd578c2ae0ea3c13d1a4565=hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565., 8fa582fb6c7a21404f4927f1ac2befef=testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef.} 2024-11-25T03:58:38,094 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 74fcbd80d22284921c26f33def236beb, 7cc8c925e00abed8a7bf0e5438157cb7, 8fa582fb6c7a21404f4927f1ac2befef, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:38,295 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 74fcbd80d22284921c26f33def236beb, 7cc8c925e00abed8a7bf0e5438157cb7, 8fa582fb6c7a21404f4927f1ac2befef, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:38,400 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7cc8c925e00abed8a7bf0e5438157cb7/.tmp/testFamily/745ea443a1934f50a4ac4ad8725b4cfb 2024-11-25T03:58:38,414 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7cc8c925e00abed8a7bf0e5438157cb7/.tmp/testFamily/745ea443a1934f50a4ac4ad8725b4cfb as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7cc8c925e00abed8a7bf0e5438157cb7/testFamily/745ea443a1934f50a4ac4ad8725b4cfb 2024-11-25T03:58:38,422 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7cc8c925e00abed8a7bf0e5438157cb7/testFamily/745ea443a1934f50a4ac4ad8725b4cfb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:58:38,423 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7cc8c925e00abed8a7bf0e5438157cb7 in 447ms, sequenceid=8, compaction requested=false 2024-11-25T03:58:38,428 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7cc8c925e00abed8a7bf0e5438157cb7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:58:38,428 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. 2024-11-25T03:58:38,428 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7cc8c925e00abed8a7bf0e5438157cb7: Waiting for close lock at 1732507117976Running coprocessor pre-close hooks at 1732507117976Disabling compacts and flushes for region at 1732507117976Disabling writes for close at 1732507117976Obtaining lock to block concurrent updates at 1732507117976Preparing flush snapshotting stores in 7cc8c925e00abed8a7bf0e5438157cb7 at 1732507117976Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732507117977 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. at 1732507117977Flushing 7cc8c925e00abed8a7bf0e5438157cb7/testFamily: creating writer at 1732507117977Flushing 7cc8c925e00abed8a7bf0e5438157cb7/testFamily: appending metadata at 1732507117992 (+15 ms)Flushing 7cc8c925e00abed8a7bf0e5438157cb7/testFamily: closing flushed file at 1732507117992Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5973b54: reopening flushed file at 1732507118413 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7cc8c925e00abed8a7bf0e5438157cb7 in 447ms, sequenceid=8, compaction requested=false at 1732507118423 (+10 ms)Writing region close event to WAL at 1732507118424 (+1 ms)Running coprocessor post-close hooks at 1732507118428 (+4 ms)Closed at 1732507118428 2024-11-25T03:58:38,428 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732507050920.7cc8c925e00abed8a7bf0e5438157cb7. 2024-11-25T03:58:38,429 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 74fcbd80d22284921c26f33def236beb, disabling compactions & flushes 2024-11-25T03:58:38,429 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. 2024-11-25T03:58:38,429 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. 2024-11-25T03:58:38,429 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. after waiting 0 ms 2024-11-25T03:58:38,429 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. 2024-11-25T03:58:38,429 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 74fcbd80d22284921c26f33def236beb 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:58:38,446 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScannerAsync/74fcbd80d22284921c26f33def236beb/.tmp/testFamily/2ef5837114dd4039b199805d89200b31 is 61, key is row-0/testFamily:testQualifier/1732507113001/Put/seqid=0 2024-11-25T03:58:38,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741863_1039 (size=5049) 2024-11-25T03:58:38,495 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 74fcbd80d22284921c26f33def236beb, 8fa582fb6c7a21404f4927f1ac2befef, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:38,696 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 74fcbd80d22284921c26f33def236beb, 8fa582fb6c7a21404f4927f1ac2befef, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:38,853 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScannerAsync/74fcbd80d22284921c26f33def236beb/.tmp/testFamily/2ef5837114dd4039b199805d89200b31 2024-11-25T03:58:38,870 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScannerAsync/74fcbd80d22284921c26f33def236beb/.tmp/testFamily/2ef5837114dd4039b199805d89200b31 as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScannerAsync/74fcbd80d22284921c26f33def236beb/testFamily/2ef5837114dd4039b199805d89200b31 2024-11-25T03:58:38,878 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScannerAsync/74fcbd80d22284921c26f33def236beb/testFamily/2ef5837114dd4039b199805d89200b31, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:58:38,879 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 74fcbd80d22284921c26f33def236beb in 450ms, sequenceid=8, compaction requested=false 2024-11-25T03:58:38,883 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnOpenScannerAsync/74fcbd80d22284921c26f33def236beb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:58:38,884 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. 2024-11-25T03:58:38,884 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 74fcbd80d22284921c26f33def236beb: Waiting for close lock at 1732507118429Running coprocessor pre-close hooks at 1732507118429Disabling compacts and flushes for region at 1732507118429Disabling writes for close at 1732507118429Obtaining lock to block concurrent updates at 1732507118429Preparing flush snapshotting stores in 74fcbd80d22284921c26f33def236beb at 1732507118429Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732507118429Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. at 1732507118430 (+1 ms)Flushing 74fcbd80d22284921c26f33def236beb/testFamily: creating writer at 1732507118430Flushing 74fcbd80d22284921c26f33def236beb/testFamily: appending metadata at 1732507118445 (+15 ms)Flushing 74fcbd80d22284921c26f33def236beb/testFamily: closing flushed file at 1732507118445Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d16ab56: reopening flushed file at 1732507118869 (+424 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 74fcbd80d22284921c26f33def236beb in 450ms, sequenceid=8, compaction requested=false at 1732507118879 (+10 ms)Writing region close event to WAL at 1732507118879Running coprocessor post-close hooks at 1732507118884 (+5 ms)Closed at 1732507118884 2024-11-25T03:58:38,884 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732507109940.74fcbd80d22284921c26f33def236beb. 2024-11-25T03:58:38,884 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 72ee6791c3678080578ac06256642fd0, disabling compactions & flushes 2024-11-25T03:58:38,884 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. 2024-11-25T03:58:38,884 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. 2024-11-25T03:58:38,885 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. after waiting 0 ms 2024-11-25T03:58:38,885 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. 2024-11-25T03:58:38,885 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 72ee6791c3678080578ac06256642fd0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:58:38,896 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 8fa582fb6c7a21404f4927f1ac2befef, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:38,900 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNext/72ee6791c3678080578ac06256642fd0/.tmp/testFamily/d2f7ccc6d27d44e0a51df76176c15cc9 is 61, key is row-0/testFamily:testQualifier/1732507090741/Put/seqid=0 2024-11-25T03:58:38,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741864_1040 (size=5049) 2024-11-25T03:58:39,096 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-25T03:58:39,097 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1325): Online Regions={72ee6791c3678080578ac06256642fd0=testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0., 2d3b244c1794e17d8de0df895f56030d=hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d., fee51fafcdd578c2ae0ea3c13d1a4565=hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565., 8fa582fb6c7a21404f4927f1ac2befef=testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef.} 2024-11-25T03:58:39,097 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 8fa582fb6c7a21404f4927f1ac2befef, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:39,298 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 2d3b244c1794e17d8de0df895f56030d, 72ee6791c3678080578ac06256642fd0, 8fa582fb6c7a21404f4927f1ac2befef, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:39,307 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNext/72ee6791c3678080578ac06256642fd0/.tmp/testFamily/d2f7ccc6d27d44e0a51df76176c15cc9 2024-11-25T03:58:39,323 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNext/72ee6791c3678080578ac06256642fd0/.tmp/testFamily/d2f7ccc6d27d44e0a51df76176c15cc9 as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNext/72ee6791c3678080578ac06256642fd0/testFamily/d2f7ccc6d27d44e0a51df76176c15cc9 2024-11-25T03:58:39,330 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNext/72ee6791c3678080578ac06256642fd0/testFamily/d2f7ccc6d27d44e0a51df76176c15cc9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:58:39,331 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 72ee6791c3678080578ac06256642fd0 in 446ms, sequenceid=8, compaction requested=false 2024-11-25T03:58:39,336 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testNormalScanTimeoutOnNext/72ee6791c3678080578ac06256642fd0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:58:39,337 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. 2024-11-25T03:58:39,337 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 72ee6791c3678080578ac06256642fd0: Waiting for close lock at 1732507118884Running coprocessor pre-close hooks at 1732507118884Disabling compacts and flushes for region at 1732507118884Disabling writes for close at 1732507118885 (+1 ms)Obtaining lock to block concurrent updates at 1732507118885Preparing flush snapshotting stores in 72ee6791c3678080578ac06256642fd0 at 1732507118885Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732507118885Flushing stores of testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. at 1732507118885Flushing 72ee6791c3678080578ac06256642fd0/testFamily: creating writer at 1732507118886 (+1 ms)Flushing 72ee6791c3678080578ac06256642fd0/testFamily: appending metadata at 1732507118900 (+14 ms)Flushing 72ee6791c3678080578ac06256642fd0/testFamily: closing flushed file at 1732507118900Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fe2e39f: reopening flushed file at 1732507119322 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 72ee6791c3678080578ac06256642fd0 in 446ms, sequenceid=8, compaction requested=false at 1732507119331 (+9 ms)Writing region close event to WAL at 1732507119332 (+1 ms)Running coprocessor post-close hooks at 1732507119337 (+5 ms)Closed at 1732507119337 2024-11-25T03:58:39,337 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732507087690.72ee6791c3678080578ac06256642fd0. 2024-11-25T03:58:39,338 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2d3b244c1794e17d8de0df895f56030d, disabling compactions & flushes 2024-11-25T03:58:39,338 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. 2024-11-25T03:58:39,338 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. 2024-11-25T03:58:39,338 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. after waiting 0 ms 2024-11-25T03:58:39,338 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. 2024-11-25T03:58:39,338 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2d3b244c1794e17d8de0df895f56030d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:58:39,353 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScanner/2d3b244c1794e17d8de0df895f56030d/.tmp/testFamily/d2f425e459664e5bbc77f027a02f16b6 is 61, key is row-0/testFamily:testQualifier/1732507057171/Put/seqid=0 2024-11-25T03:58:39,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741865_1041 (size=5049) 2024-11-25T03:58:39,498 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 2d3b244c1794e17d8de0df895f56030d, 8fa582fb6c7a21404f4927f1ac2befef, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:39,698 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 2d3b244c1794e17d8de0df895f56030d, 8fa582fb6c7a21404f4927f1ac2befef, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:39,759 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScanner/2d3b244c1794e17d8de0df895f56030d/.tmp/testFamily/d2f425e459664e5bbc77f027a02f16b6 2024-11-25T03:58:39,772 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScanner/2d3b244c1794e17d8de0df895f56030d/.tmp/testFamily/d2f425e459664e5bbc77f027a02f16b6 as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScanner/2d3b244c1794e17d8de0df895f56030d/testFamily/d2f425e459664e5bbc77f027a02f16b6 2024-11-25T03:58:39,780 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScanner/2d3b244c1794e17d8de0df895f56030d/testFamily/d2f425e459664e5bbc77f027a02f16b6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:58:39,781 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2d3b244c1794e17d8de0df895f56030d in 443ms, sequenceid=8, compaction requested=false 2024-11-25T03:58:39,786 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnOpenScanner/2d3b244c1794e17d8de0df895f56030d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:58:39,787 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. 2024-11-25T03:58:39,787 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2d3b244c1794e17d8de0df895f56030d: Waiting for close lock at 1732507119338Running coprocessor pre-close hooks at 1732507119338Disabling compacts and flushes for region at 1732507119338Disabling writes for close at 1732507119338Obtaining lock to block concurrent updates at 1732507119338Preparing flush snapshotting stores in 2d3b244c1794e17d8de0df895f56030d at 1732507119338Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732507119338Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. at 1732507119339 (+1 ms)Flushing 2d3b244c1794e17d8de0df895f56030d/testFamily: creating writer at 1732507119339Flushing 2d3b244c1794e17d8de0df895f56030d/testFamily: appending metadata at 1732507119353 (+14 ms)Flushing 2d3b244c1794e17d8de0df895f56030d/testFamily: closing flushed file at 1732507119353Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e073f54: reopening flushed file at 1732507119771 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2d3b244c1794e17d8de0df895f56030d in 443ms, sequenceid=8, compaction requested=false at 1732507119781 (+10 ms)Writing region close event to WAL at 1732507119782 (+1 ms)Running coprocessor post-close hooks at 1732507119787 (+5 ms)Closed at 1732507119787 2024-11-25T03:58:39,787 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732507054121.2d3b244c1794e17d8de0df895f56030d. 2024-11-25T03:58:39,788 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fee51fafcdd578c2ae0ea3c13d1a4565, disabling compactions & flushes 2024-11-25T03:58:39,788 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. 2024-11-25T03:58:39,788 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. 2024-11-25T03:58:39,788 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. after waiting 0 ms 2024-11-25T03:58:39,788 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. 2024-11-25T03:58:39,788 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fee51fafcdd578c2ae0ea3c13d1a4565 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:58:39,805 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNextAsync/fee51fafcdd578c2ae0ea3c13d1a4565/.tmp/testFamily/b7b65a030cc34e01a709fe92a615358a is 61, key is row-0/testFamily:testQualifier/1732507066267/Put/seqid=0 2024-11-25T03:58:39,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741866_1042 (size=5049) 2024-11-25T03:58:39,899 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 8fa582fb6c7a21404f4927f1ac2befef, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:40,099 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T03:58:40,100 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1325): Online Regions={fee51fafcdd578c2ae0ea3c13d1a4565=hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565., 8fa582fb6c7a21404f4927f1ac2befef=testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef.} 2024-11-25T03:58:40,100 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 8fa582fb6c7a21404f4927f1ac2befef, fee51fafcdd578c2ae0ea3c13d1a4565 2024-11-25T03:58:40,212 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNextAsync/fee51fafcdd578c2ae0ea3c13d1a4565/.tmp/testFamily/b7b65a030cc34e01a709fe92a615358a 2024-11-25T03:58:40,226 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNextAsync/fee51fafcdd578c2ae0ea3c13d1a4565/.tmp/testFamily/b7b65a030cc34e01a709fe92a615358a as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNextAsync/fee51fafcdd578c2ae0ea3c13d1a4565/testFamily/b7b65a030cc34e01a709fe92a615358a 2024-11-25T03:58:40,233 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNextAsync/fee51fafcdd578c2ae0ea3c13d1a4565/testFamily/b7b65a030cc34e01a709fe92a615358a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:58:40,234 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fee51fafcdd578c2ae0ea3c13d1a4565 in 446ms, sequenceid=8, compaction requested=false 2024-11-25T03:58:40,239 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/hbase/testMetaScanTimeoutOnNextAsync/fee51fafcdd578c2ae0ea3c13d1a4565/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:58:40,239 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. 2024-11-25T03:58:40,239 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fee51fafcdd578c2ae0ea3c13d1a4565: Waiting for close lock at 1732507119788Running coprocessor pre-close hooks at 1732507119788Disabling compacts and flushes for region at 1732507119788Disabling writes for close at 1732507119788Obtaining lock to block concurrent updates at 1732507119788Preparing flush snapshotting stores in fee51fafcdd578c2ae0ea3c13d1a4565 at 1732507119788Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732507119788Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. at 1732507119789 (+1 ms)Flushing fee51fafcdd578c2ae0ea3c13d1a4565/testFamily: creating writer at 1732507119789Flushing fee51fafcdd578c2ae0ea3c13d1a4565/testFamily: appending metadata at 1732507119804 (+15 ms)Flushing fee51fafcdd578c2ae0ea3c13d1a4565/testFamily: closing flushed file at 1732507119804Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7988b934: reopening flushed file at 1732507120224 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fee51fafcdd578c2ae0ea3c13d1a4565 in 446ms, sequenceid=8, compaction requested=false at 1732507120234 (+10 ms)Writing region close event to WAL at 1732507120235 (+1 ms)Running coprocessor post-close hooks at 1732507120239 (+4 ms)Closed at 1732507120239 2024-11-25T03:58:40,239 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732507063215.fee51fafcdd578c2ae0ea3c13d1a4565. 2024-11-25T03:58:40,240 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8fa582fb6c7a21404f4927f1ac2befef, disabling compactions & flushes 2024-11-25T03:58:40,240 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. 2024-11-25T03:58:40,240 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. 2024-11-25T03:58:40,240 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. after waiting 0 ms 2024-11-25T03:58:40,240 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. 2024-11-25T03:58:40,240 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8fa582fb6c7a21404f4927f1ac2befef 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:58:40,254 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextException/8fa582fb6c7a21404f4927f1ac2befef/.tmp/testFamily/a77c5e3652e04b91ad151e944477d58e is 61, key is row-0/testFamily:testQualifier/1732507075382/Put/seqid=0 2024-11-25T03:58:40,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741867_1043 (size=5049) 2024-11-25T03:58:40,300 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 8fa582fb6c7a21404f4927f1ac2befef 2024-11-25T03:58:40,501 DEBUG [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1351): Waiting on 8fa582fb6c7a21404f4927f1ac2befef 2024-11-25T03:58:40,662 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextException/8fa582fb6c7a21404f4927f1ac2befef/.tmp/testFamily/a77c5e3652e04b91ad151e944477d58e 2024-11-25T03:58:40,676 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextException/8fa582fb6c7a21404f4927f1ac2befef/.tmp/testFamily/a77c5e3652e04b91ad151e944477d58e as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextException/8fa582fb6c7a21404f4927f1ac2befef/testFamily/a77c5e3652e04b91ad151e944477d58e 2024-11-25T03:58:40,682 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextException/8fa582fb6c7a21404f4927f1ac2befef/testFamily/a77c5e3652e04b91ad151e944477d58e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:58:40,684 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8fa582fb6c7a21404f4927f1ac2befef in 444ms, sequenceid=8, compaction requested=false 2024-11-25T03:58:40,689 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/data/default/testRetryOutOfOrderScannerNextException/8fa582fb6c7a21404f4927f1ac2befef/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:58:40,689 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. 2024-11-25T03:58:40,690 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8fa582fb6c7a21404f4927f1ac2befef: Waiting for close lock at 1732507120240Running coprocessor pre-close hooks at 1732507120240Disabling compacts and flushes for region at 1732507120240Disabling writes for close at 1732507120240Obtaining lock to block concurrent updates at 1732507120240Preparing flush snapshotting stores in 8fa582fb6c7a21404f4927f1ac2befef at 1732507120240Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732507120240Flushing stores of testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. at 1732507120241 (+1 ms)Flushing 8fa582fb6c7a21404f4927f1ac2befef/testFamily: creating writer at 1732507120241Flushing 8fa582fb6c7a21404f4927f1ac2befef/testFamily: appending metadata at 1732507120254 (+13 ms)Flushing 8fa582fb6c7a21404f4927f1ac2befef/testFamily: closing flushed file at 1732507120254Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b45bedb: reopening flushed file at 1732507120675 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8fa582fb6c7a21404f4927f1ac2befef in 444ms, sequenceid=8, compaction requested=false at 1732507120684 (+9 ms)Writing region close event to WAL at 1732507120684Running coprocessor post-close hooks at 1732507120689 (+5 ms)Closed at 1732507120689 2024-11-25T03:58:40,690 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732507072313.8fa582fb6c7a21404f4927f1ac2befef. 2024-11-25T03:58:40,701 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(976): stopping server d73269e94d72,33579,1732507016762; all regions closed. 2024-11-25T03:58:40,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741834_1010 (size=19479) 2024-11-25T03:58:40,711 DEBUG [RS:0;d73269e94d72:33579 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/oldWALs 2024-11-25T03:58:40,711 INFO [RS:0;d73269e94d72:33579 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C33579%2C1732507016762.meta:.meta(num 1732507050433) 2024-11-25T03:58:40,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741833_1009 (size=20031) 2024-11-25T03:58:40,714 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/WALs/d73269e94d72,33579,1732507016762/d73269e94d72%2C33579%2C1732507016762.1732507049961 not finished, retry = 0 2024-11-25T03:58:40,822 DEBUG [RS:0;d73269e94d72:33579 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/oldWALs 2024-11-25T03:58:40,822 INFO [RS:0;d73269e94d72:33579 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C33579%2C1732507016762:(num 1732507049961) 2024-11-25T03:58:40,822 DEBUG [RS:0;d73269e94d72:33579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:58:40,822 INFO [RS:0;d73269e94d72:33579 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:58:40,823 INFO [RS:0;d73269e94d72:33579 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:58:40,823 INFO [RS:0;d73269e94d72:33579 {}] hbase.ChoreService(370): Chore service for: regionserver/d73269e94d72:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T03:58:40,824 INFO [RS:0;d73269e94d72:33579 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:58:40,824 INFO [regionserver/d73269e94d72:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:58:40,826 INFO [RS:0;d73269e94d72:33579 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33579 2024-11-25T03:58:40,828 INFO [regionserver/d73269e94d72:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:58:40,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:58:40,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d73269e94d72,33579,1732507016762 2024-11-25T03:58:40,838 INFO [RS:0;d73269e94d72:33579 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:58:40,849 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d73269e94d72,33579,1732507016762] 2024-11-25T03:58:40,859 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d73269e94d72,33579,1732507016762 already deleted, retry=false 2024-11-25T03:58:40,859 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d73269e94d72,33579,1732507016762 expired; onlineServers=0 2024-11-25T03:58:40,859 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd73269e94d72,38513,1732507015876' ***** 2024-11-25T03:58:40,859 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T03:58:40,860 INFO [M:0;d73269e94d72:38513 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:58:40,860 INFO [M:0;d73269e94d72:38513 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:58:40,860 DEBUG [M:0;d73269e94d72:38513 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T03:58:40,860 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T03:58:40,860 DEBUG [M:0;d73269e94d72:38513 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T03:58:40,860 DEBUG [master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732507019742 {}] cleaner.HFileCleaner(306): Exit Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732507019742,5,FailOnTimeoutGroup] 2024-11-25T03:58:40,860 DEBUG [master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732507019743 {}] cleaner.HFileCleaner(306): Exit Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732507019743,5,FailOnTimeoutGroup] 2024-11-25T03:58:40,861 INFO [M:0;d73269e94d72:38513 {}] hbase.ChoreService(370): Chore service for: master/d73269e94d72:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T03:58:40,861 INFO [M:0;d73269e94d72:38513 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:58:40,861 DEBUG [M:0;d73269e94d72:38513 {}] master.HMaster(1795): Stopping service threads 2024-11-25T03:58:40,861 INFO [M:0;d73269e94d72:38513 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T03:58:40,862 INFO [M:0;d73269e94d72:38513 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:58:40,862 INFO [M:0;d73269e94d72:38513 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:58:40,863 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T03:58:40,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T03:58:40,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:58:40,870 DEBUG [M:0;d73269e94d72:38513 {}] zookeeper.ZKUtil(347): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T03:58:40,870 WARN [M:0;d73269e94d72:38513 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T03:58:40,872 INFO [M:0;d73269e94d72:38513 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/.lastflushedseqids 2024-11-25T03:58:40,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741868_1044 (size=649) 2024-11-25T03:58:40,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:58:40,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33579-0x101702312f90001, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:58:40,950 INFO [RS:0;d73269e94d72:33579 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:58:40,951 INFO [RS:0;d73269e94d72:33579 {}] regionserver.HRegionServer(1031): Exiting; stopping=d73269e94d72,33579,1732507016762; zookeeper connection closed. 2024-11-25T03:58:40,952 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@d6fb26 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@d6fb26 2024-11-25T03:58:40,953 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-25T03:58:41,289 INFO [M:0;d73269e94d72:38513 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:58:41,290 INFO [M:0;d73269e94d72:38513 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T03:58:41,291 DEBUG [M:0;d73269e94d72:38513 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:58:41,291 INFO [M:0;d73269e94d72:38513 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:58:41,291 DEBUG [M:0;d73269e94d72:38513 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:58:41,291 DEBUG [M:0;d73269e94d72:38513 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:58:41,292 DEBUG [M:0;d73269e94d72:38513 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:58:41,292 INFO [M:0;d73269e94d72:38513 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.08 KB heapSize=190.98 KB 2024-11-25T03:58:41,315 DEBUG [M:0;d73269e94d72:38513 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f3e9f146b7144a9fa5b32b9a949f0be2 is 82, key is hbase:meta,,1/info:regioninfo/1732507050515/Put/seqid=0 2024-11-25T03:58:41,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741869_1045 (size=5672) 2024-11-25T03:58:41,721 INFO [M:0;d73269e94d72:38513 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f3e9f146b7144a9fa5b32b9a949f0be2 2024-11-25T03:58:41,753 DEBUG [M:0;d73269e94d72:38513 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/72fb2ebb9b4a43d0b3413c20d87b6b78 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732507052188/Put/seqid=0 2024-11-25T03:58:41,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741870_1046 (size=14078) 2024-11-25T03:58:42,135 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:58:42,158 INFO [M:0;d73269e94d72:38513 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.53 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/72fb2ebb9b4a43d0b3413c20d87b6b78 2024-11-25T03:58:42,165 INFO [M:0;d73269e94d72:38513 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 72fb2ebb9b4a43d0b3413c20d87b6b78 2024-11-25T03:58:42,186 DEBUG [M:0;d73269e94d72:38513 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0b8b0366e7824ba4958e6a1ae695d2fe is 69, key is d73269e94d72,33579,1732507016762/rs:state/1732507019796/Put/seqid=0 2024-11-25T03:58:42,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741871_1047 (size=5156) 2024-11-25T03:58:42,593 INFO [M:0;d73269e94d72:38513 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0b8b0366e7824ba4958e6a1ae695d2fe 2024-11-25T03:58:42,607 DEBUG [M:0;d73269e94d72:38513 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f3e9f146b7144a9fa5b32b9a949f0be2 as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f3e9f146b7144a9fa5b32b9a949f0be2 2024-11-25T03:58:42,613 INFO [M:0;d73269e94d72:38513 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f3e9f146b7144a9fa5b32b9a949f0be2, entries=8, sequenceid=317, filesize=5.5 K 2024-11-25T03:58:42,614 DEBUG [M:0;d73269e94d72:38513 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/72fb2ebb9b4a43d0b3413c20d87b6b78 as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/72fb2ebb9b4a43d0b3413c20d87b6b78 2024-11-25T03:58:42,620 INFO [M:0;d73269e94d72:38513 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 72fb2ebb9b4a43d0b3413c20d87b6b78 2024-11-25T03:58:42,621 INFO [M:0;d73269e94d72:38513 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/72fb2ebb9b4a43d0b3413c20d87b6b78, entries=33, sequenceid=317, filesize=13.7 K 2024-11-25T03:58:42,622 DEBUG [M:0;d73269e94d72:38513 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0b8b0366e7824ba4958e6a1ae695d2fe as hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0b8b0366e7824ba4958e6a1ae695d2fe 2024-11-25T03:58:42,629 INFO [M:0;d73269e94d72:38513 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39991/user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0b8b0366e7824ba4958e6a1ae695d2fe, entries=1, sequenceid=317, filesize=5.0 K 2024-11-25T03:58:42,630 INFO [M:0;d73269e94d72:38513 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.08 KB/166999, heapSize ~190.68 KB/195256, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1338ms, sequenceid=317, compaction requested=false 2024-11-25T03:58:42,632 INFO [M:0;d73269e94d72:38513 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:58:42,632 DEBUG [M:0;d73269e94d72:38513 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732507121291Disabling compacts and flushes for region at 1732507121291Disabling writes for close at 1732507121291Obtaining lock to block concurrent updates at 1732507121292 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732507121292Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166999, getHeapSize=195496, getOffHeapSize=0, getCellsCount=364 at 1732507121294 (+2 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732507121296 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732507121296Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732507121314 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732507121314Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732507121734 (+420 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732507121752 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732507121752Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732507122166 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732507122185 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732507122185Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b110bd5: reopening flushed file at 1732507122605 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a5a9cee: reopening flushed file at 1732507122613 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@426a9e36: reopening flushed file at 1732507122621 (+8 ms)Finished flush of dataSize ~163.08 KB/166999, heapSize ~190.68 KB/195256, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1338ms, sequenceid=317, compaction requested=false at 1732507122630 (+9 ms)Writing region close event to WAL at 1732507122632 (+2 ms)Closed at 1732507122632 2024-11-25T03:58:42,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36505 is added to blk_1073741830_1006 (size=188211) 2024-11-25T03:58:42,635 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/0d89b72b-558f-e655-c468-04b8fecf5fcc/MasterData/WALs/d73269e94d72,38513,1732507015876/d73269e94d72%2C38513%2C1732507015876.1732507018966 not finished, retry = 0 2024-11-25T03:58:42,736 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:58:42,736 INFO [M:0;d73269e94d72:38513 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T03:58:42,736 INFO [M:0;d73269e94d72:38513 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38513 2024-11-25T03:58:42,737 INFO [M:0;d73269e94d72:38513 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:58:42,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:58:42,850 INFO [M:0;d73269e94d72:38513 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:58:42,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38513-0x101702312f90000, quorum=127.0.0.1:50296, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:58:42,893 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:58:42,898 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T03:58:42,898 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T03:58:42,898 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T03:58:42,899 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/hadoop.log.dir/,STOPPED} 2024-11-25T03:58:42,902 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T03:58:42,902 WARN [BP-823003919-172.17.0.2-1732507011402 heartbeating to localhost/127.0.0.1:39991 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T03:58:42,902 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T03:58:42,902 WARN [BP-823003919-172.17.0.2-1732507011402 heartbeating to localhost/127.0.0.1:39991 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-823003919-172.17.0.2-1732507011402 (Datanode Uuid 08cc274d-cd7c-426f-a1e2-53d88e7c2d05) service to localhost/127.0.0.1:39991 2024-11-25T03:58:42,903 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/cluster_0e4f7757-e78d-eac9-4af1-03483e95706d/data/data1/current/BP-823003919-172.17.0.2-1732507011402 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T03:58:42,903 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/cluster_0e4f7757-e78d-eac9-4af1-03483e95706d/data/data2/current/BP-823003919-172.17.0.2-1732507011402 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T03:58:42,903 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T03:58:42,913 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T03:58:42,914 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T03:58:42,914 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T03:58:42,915 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T03:58:42,915 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9a48a856-ad0e-fd6d-8c74-726d0095d6d7/hadoop.log.dir/,STOPPED} 2024-11-25T03:58:42,921 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T03:58:42,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down