2024-11-11 23:54:46,665 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-11 23:54:46,696 main DEBUG Took 0.029289 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 23:54:46,697 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 23:54:46,697 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 23:54:46,698 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 23:54:46,699 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:54:46,708 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 23:54:46,725 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:54:46,727 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:54:46,728 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:54:46,728 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:54:46,729 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:54:46,730 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:54:46,731 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:54:46,732 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:54:46,733 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:54:46,733 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:54:46,734 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:54:46,736 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:54:46,737 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:54:46,738 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:54:46,738 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:54:46,739 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:54:46,739 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:54:46,740 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:54:46,740 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:54:46,741 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:54:46,741 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:54:46,742 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:54:46,743 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:54:46,743 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 23:54:46,744 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:54:46,744 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 23:54:46,746 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 23:54:46,748 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 23:54:46,750 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 23:54:46,751 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 23:54:46,753 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 23:54:46,753 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 23:54:46,764 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 23:54:46,767 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 23:54:46,769 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 23:54:46,770 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 23:54:46,770 main DEBUG createAppenders(={Console}) 2024-11-11 23:54:46,771 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-11 23:54:46,772 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-11 23:54:46,772 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-11 23:54:46,773 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 23:54:46,773 main DEBUG OutputStream closed 2024-11-11 23:54:46,774 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 23:54:46,774 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 23:54:46,774 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-11 23:54:46,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 23:54:46,893 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 23:54:46,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 23:54:46,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 23:54:46,897 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 23:54:46,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 23:54:46,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 23:54:46,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 23:54:46,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 23:54:46,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 23:54:46,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 23:54:46,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 23:54:46,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 23:54:46,902 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 23:54:46,902 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 23:54:46,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 23:54:46,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 23:54:46,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 23:54:46,907 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 23:54:46,907 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-11 23:54:46,907 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 23:54:46,908 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-11T23:54:46,920 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-11 23:54:46,923 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 23:54:46,924 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T23:54:47,257 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1 2024-11-11T23:54:47,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T23:54:47,306 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/cluster_2dda0b1b-20c8-cfdb-81b8-35b1b1adfbd5, deleteOnExit=true 2024-11-11T23:54:47,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T23:54:47,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/test.cache.data in system properties and HBase conf 2024-11-11T23:54:47,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T23:54:47,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/hadoop.log.dir in system properties and HBase conf 2024-11-11T23:54:47,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T23:54:47,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T23:54:47,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T23:54:47,428 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T23:54:47,564 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T23:54:47,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T23:54:47,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T23:54:47,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T23:54:47,572 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T23:54:47,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T23:54:47,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T23:54:47,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T23:54:47,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T23:54:47,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T23:54:47,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/nfs.dump.dir in system properties and HBase conf 2024-11-11T23:54:47,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/java.io.tmpdir in system properties and HBase conf 2024-11-11T23:54:47,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T23:54:47,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T23:54:47,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T23:54:48,633 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T23:54:48,713 INFO [Time-limited test {}] log.Log(170): Logging initialized @2972ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T23:54:48,798 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T23:54:48,864 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T23:54:48,883 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T23:54:48,883 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T23:54:48,884 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T23:54:48,896 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T23:54:48,899 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/hadoop.log.dir/,AVAILABLE} 2024-11-11T23:54:48,900 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T23:54:49,076 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@40b93e99{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/java.io.tmpdir/jetty-localhost-43329-hadoop-hdfs-3_4_1-tests_jar-_-any-6036979549545455871/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T23:54:49,081 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:43329} 2024-11-11T23:54:49,082 INFO [Time-limited test {}] server.Server(415): Started @3342ms 2024-11-11T23:54:49,422 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T23:54:49,431 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T23:54:49,432 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T23:54:49,432 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T23:54:49,432 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T23:54:49,433 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/hadoop.log.dir/,AVAILABLE} 2024-11-11T23:54:49,434 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T23:54:49,547 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@429b6cf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/java.io.tmpdir/jetty-localhost-46711-hadoop-hdfs-3_4_1-tests_jar-_-any-13797512574568453117/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T23:54:49,548 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:46711} 2024-11-11T23:54:49,548 INFO [Time-limited test {}] server.Server(415): Started @3808ms 2024-11-11T23:54:49,595 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T23:54:50,037 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/cluster_2dda0b1b-20c8-cfdb-81b8-35b1b1adfbd5/data/data2/current/BP-716215489-172.17.0.2-1731369288393/current, will proceed with Du for space computation calculation, 2024-11-11T23:54:50,037 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/cluster_2dda0b1b-20c8-cfdb-81b8-35b1b1adfbd5/data/data1/current/BP-716215489-172.17.0.2-1731369288393/current, will proceed with Du for space computation calculation, 2024-11-11T23:54:50,079 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T23:54:50,140 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x14fb7f116e355b8d with lease ID 0xe71c7aa31a6c3deb: Processing first storage report for DS-640312f5-10b0-40ed-9882-7da92da622b1 from datanode DatanodeRegistration(127.0.0.1:37981, datanodeUuid=58e05cbd-3d48-42e0-a8b2-a664a3b734ee, infoPort=42225, infoSecurePort=0, ipcPort=34387, storageInfo=lv=-57;cid=testClusterID;nsid=1672619291;c=1731369288393) 2024-11-11T23:54:50,141 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x14fb7f116e355b8d with lease ID 0xe71c7aa31a6c3deb: from storage DS-640312f5-10b0-40ed-9882-7da92da622b1 node DatanodeRegistration(127.0.0.1:37981, datanodeUuid=58e05cbd-3d48-42e0-a8b2-a664a3b734ee, infoPort=42225, infoSecurePort=0, ipcPort=34387, storageInfo=lv=-57;cid=testClusterID;nsid=1672619291;c=1731369288393), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T23:54:50,142 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x14fb7f116e355b8d with lease ID 0xe71c7aa31a6c3deb: Processing first storage report for DS-12c44763-7968-439c-8540-49baa830de87 from datanode DatanodeRegistration(127.0.0.1:37981, datanodeUuid=58e05cbd-3d48-42e0-a8b2-a664a3b734ee, infoPort=42225, infoSecurePort=0, ipcPort=34387, storageInfo=lv=-57;cid=testClusterID;nsid=1672619291;c=1731369288393) 2024-11-11T23:54:50,142 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x14fb7f116e355b8d with lease ID 0xe71c7aa31a6c3deb: from storage DS-12c44763-7968-439c-8540-49baa830de87 node DatanodeRegistration(127.0.0.1:37981, datanodeUuid=58e05cbd-3d48-42e0-a8b2-a664a3b734ee, infoPort=42225, infoSecurePort=0, ipcPort=34387, storageInfo=lv=-57;cid=testClusterID;nsid=1672619291;c=1731369288393), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T23:54:50,168 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1 2024-11-11T23:54:50,242 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/cluster_2dda0b1b-20c8-cfdb-81b8-35b1b1adfbd5/zookeeper_0, clientPort=56838, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/cluster_2dda0b1b-20c8-cfdb-81b8-35b1b1adfbd5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/cluster_2dda0b1b-20c8-cfdb-81b8-35b1b1adfbd5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T23:54:50,253 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56838 2024-11-11T23:54:50,267 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T23:54:50,272 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T23:54:50,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741825_1001 (size=7) 2024-11-11T23:54:50,889 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018 with version=8 2024-11-11T23:54:50,890 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/hbase-staging 2024-11-11T23:54:50,971 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T23:54:51,164 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b70d29849faf:0 server-side Connection retries=9 2024-11-11T23:54:51,173 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T23:54:51,174 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T23:54:51,178 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T23:54:51,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T23:54:51,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T23:54:51,296 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T23:54:51,345 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T23:54:51,353 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T23:54:51,356 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T23:54:51,377 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 46679 (auto-detected) 2024-11-11T23:54:51,378 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T23:54:51,395 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34765 2024-11-11T23:54:51,413 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34765 connecting to ZooKeeper ensemble=127.0.0.1:56838 2024-11-11T23:54:51,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:347650x0, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T23:54:51,441 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34765-0x1003136265b0000 connected 2024-11-11T23:54:51,468 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T23:54:51,470 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T23:54:51,479 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T23:54:51,482 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018, hbase.cluster.distributed=false 2024-11-11T23:54:51,505 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T23:54:51,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34765 2024-11-11T23:54:51,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34765 2024-11-11T23:54:51,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34765 2024-11-11T23:54:51,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34765 2024-11-11T23:54:51,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34765 2024-11-11T23:54:51,619 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b70d29849faf:0 server-side Connection retries=9 2024-11-11T23:54:51,621 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T23:54:51,621 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T23:54:51,621 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T23:54:51,621 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T23:54:51,622 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T23:54:51,624 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T23:54:51,627 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T23:54:51,628 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41471 2024-11-11T23:54:51,631 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41471 connecting to ZooKeeper ensemble=127.0.0.1:56838 2024-11-11T23:54:51,633 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T23:54:51,637 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T23:54:51,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:414710x0, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T23:54:51,644 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:414710x0, quorum=127.0.0.1:56838, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T23:54:51,645 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41471-0x1003136265b0001 connected 2024-11-11T23:54:51,648 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T23:54:51,657 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T23:54:51,659 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T23:54:51,664 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T23:54:51,665 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41471 2024-11-11T23:54:51,666 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41471 2024-11-11T23:54:51,669 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41471 2024-11-11T23:54:51,669 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41471 2024-11-11T23:54:51,670 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41471 2024-11-11T23:54:51,684 DEBUG [M:0;b70d29849faf:34765 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b70d29849faf:34765 2024-11-11T23:54:51,685 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b70d29849faf,34765,1731369291015 2024-11-11T23:54:51,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T23:54:51,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T23:54:51,692 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b70d29849faf,34765,1731369291015 2024-11-11T23:54:51,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T23:54:51,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:54:51,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:54:51,713 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T23:54:51,714 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b70d29849faf,34765,1731369291015 from backup master directory 2024-11-11T23:54:51,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b70d29849faf,34765,1731369291015 2024-11-11T23:54:51,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T23:54:51,717 WARN [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T23:54:51,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T23:54:51,717 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b70d29849faf,34765,1731369291015 2024-11-11T23:54:51,719 INFO [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T23:54:51,721 INFO [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T23:54:51,780 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/hbase.id] with ID: 0c7838f7-a8db-4c18-9e8a-3281aa44d78f 2024-11-11T23:54:51,780 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/.tmp/hbase.id 2024-11-11T23:54:51,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741826_1002 (size=42) 2024-11-11T23:54:52,197 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/.tmp/hbase.id]:[hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/hbase.id] 2024-11-11T23:54:52,244 INFO [master/b70d29849faf:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T23:54:52,250 INFO [master/b70d29849faf:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T23:54:52,267 INFO [master/b70d29849faf:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-11T23:54:52,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:54:52,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:54:52,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741827_1003 (size=196) 2024-11-11T23:54:52,709 INFO [master/b70d29849faf:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:54:52,712 INFO [master/b70d29849faf:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T23:54:52,725 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:54:52,728 INFO [master/b70d29849faf:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T23:54:52,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741828_1004 (size=1189) 2024-11-11T23:54:53,179 INFO [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store 2024-11-11T23:54:53,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741829_1005 (size=34) 2024-11-11T23:54:53,608 INFO [master/b70d29849faf:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T23:54:53,612 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:54:53,613 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T23:54:53,613 INFO [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:54:53,613 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:54:53,614 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T23:54:53,615 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:54:53,615 INFO [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:54:53,616 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731369293613Disabling compacts and flushes for region at 1731369293613Disabling writes for close at 1731369293615 (+2 ms)Writing region close event to WAL at 1731369293615Closed at 1731369293615 2024-11-11T23:54:53,618 WARN [master/b70d29849faf:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/.initializing 2024-11-11T23:54:53,618 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/WALs/b70d29849faf,34765,1731369291015 2024-11-11T23:54:53,625 INFO [master/b70d29849faf:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T23:54:53,639 INFO [master/b70d29849faf:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b70d29849faf%2C34765%2C1731369291015, suffix=, logDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/WALs/b70d29849faf,34765,1731369291015, archiveDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/oldWALs, maxLogs=10 2024-11-11T23:54:53,660 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/WALs/b70d29849faf,34765,1731369291015/b70d29849faf%2C34765%2C1731369291015.1731369293643, exclude list is [], retry=0 2024-11-11T23:54:53,675 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37981,DS-640312f5-10b0-40ed-9882-7da92da622b1,DISK] 2024-11-11T23:54:53,677 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T23:54:53,707 INFO [master/b70d29849faf:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/WALs/b70d29849faf,34765,1731369291015/b70d29849faf%2C34765%2C1731369291015.1731369293643 2024-11-11T23:54:53,708 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42225:42225)] 2024-11-11T23:54:53,709 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:54:53,709 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:54:53,712 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:54:53,713 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:54:53,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:54:53,767 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T23:54:53,770 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:53,772 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:54:53,773 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:54:53,776 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T23:54:53,776 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:53,777 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:54:53,777 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:54:53,780 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T23:54:53,780 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:53,781 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:54:53,782 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:54:53,784 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T23:54:53,784 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:53,785 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:54:53,786 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:54:53,789 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:54:53,790 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:54:53,795 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:54:53,796 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:54:53,799 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T23:54:53,803 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T23:54:53,807 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:54:53,808 INFO [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66629802, jitterRate=-0.007138580083847046}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T23:54:53,815 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731369293724Initializing all the Stores at 1731369293726 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731369293727 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369293727Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369293728 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369293728Cleaning up temporary data from old regions at 1731369293796 (+68 ms)Region opened successfully at 1731369293815 (+19 ms) 2024-11-11T23:54:53,817 INFO [master/b70d29849faf:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T23:54:53,846 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bc5543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b70d29849faf/172.17.0.2:0 2024-11-11T23:54:53,872 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T23:54:53,882 INFO [master/b70d29849faf:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T23:54:53,882 INFO [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T23:54:53,884 INFO [master/b70d29849faf:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T23:54:53,886 INFO [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T23:54:53,890 INFO [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T23:54:53,890 INFO [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T23:54:53,913 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T23:54:53,920 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T23:54:53,922 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T23:54:53,924 INFO [master/b70d29849faf:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T23:54:53,926 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T23:54:53,927 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T23:54:53,929 INFO [master/b70d29849faf:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T23:54:53,932 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T23:54:53,933 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T23:54:53,934 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T23:54:53,935 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T23:54:53,953 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T23:54:53,954 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T23:54:53,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T23:54:53,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T23:54:53,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:54:53,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:54:53,960 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b70d29849faf,34765,1731369291015, sessionid=0x1003136265b0000, setting cluster-up flag (Was=false) 2024-11-11T23:54:53,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:54:53,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:54:53,976 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T23:54:53,978 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b70d29849faf,34765,1731369291015 2024-11-11T23:54:53,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:54:53,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:54:53,988 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T23:54:53,990 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b70d29849faf,34765,1731369291015 2024-11-11T23:54:53,998 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T23:54:54,076 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(746): ClusterId : 0c7838f7-a8db-4c18-9e8a-3281aa44d78f 2024-11-11T23:54:54,077 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T23:54:54,078 DEBUG [RS:0;b70d29849faf:41471 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T23:54:54,082 DEBUG [RS:0;b70d29849faf:41471 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T23:54:54,082 DEBUG [RS:0;b70d29849faf:41471 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T23:54:54,085 DEBUG [RS:0;b70d29849faf:41471 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T23:54:54,085 DEBUG [RS:0;b70d29849faf:41471 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5106f025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b70d29849faf/172.17.0.2:0 2024-11-11T23:54:54,086 INFO [master/b70d29849faf:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T23:54:54,092 INFO [master/b70d29849faf:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T23:54:54,097 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b70d29849faf,34765,1731369291015 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T23:54:54,103 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b70d29849faf:41471 2024-11-11T23:54:54,104 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b70d29849faf:0, corePoolSize=5, maxPoolSize=5 2024-11-11T23:54:54,104 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b70d29849faf:0, corePoolSize=5, maxPoolSize=5 2024-11-11T23:54:54,104 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b70d29849faf:0, corePoolSize=5, maxPoolSize=5 2024-11-11T23:54:54,104 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b70d29849faf:0, corePoolSize=5, maxPoolSize=5 2024-11-11T23:54:54,104 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b70d29849faf:0, corePoolSize=10, maxPoolSize=10 2024-11-11T23:54:54,105 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:54:54,105 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b70d29849faf:0, corePoolSize=2, maxPoolSize=2 2024-11-11T23:54:54,105 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:54:54,105 INFO [RS:0;b70d29849faf:41471 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T23:54:54,105 INFO [RS:0;b70d29849faf:41471 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T23:54:54,106 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T23:54:54,107 INFO [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731369324107 2024-11-11T23:54:54,108 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(2659): reportForDuty to master=b70d29849faf,34765,1731369291015 with port=41471, startcode=1731369291585 2024-11-11T23:54:54,109 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T23:54:54,110 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T23:54:54,110 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T23:54:54,111 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T23:54:54,114 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T23:54:54,114 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T23:54:54,115 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T23:54:54,115 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T23:54:54,117 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:54,115 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T23:54:54,117 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T23:54:54,119 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T23:54:54,120 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T23:54:54,121 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T23:54:54,121 DEBUG [RS:0;b70d29849faf:41471 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T23:54:54,123 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T23:54:54,123 INFO [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T23:54:54,125 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b70d29849faf:0:becomeActiveMaster-HFileCleaner.large.0-1731369294124,5,FailOnTimeoutGroup] 2024-11-11T23:54:54,125 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b70d29849faf:0:becomeActiveMaster-HFileCleaner.small.0-1731369294125,5,FailOnTimeoutGroup] 2024-11-11T23:54:54,125 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T23:54:54,126 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T23:54:54,127 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T23:54:54,127 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T23:54:54,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741831_1007 (size=1321) 2024-11-11T23:54:54,154 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43087, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T23:54:54,160 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34765 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b70d29849faf,41471,1731369291585 2024-11-11T23:54:54,162 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34765 {}] master.ServerManager(517): Registering regionserver=b70d29849faf,41471,1731369291585 2024-11-11T23:54:54,176 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018 2024-11-11T23:54:54,176 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39869 2024-11-11T23:54:54,176 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T23:54:54,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T23:54:54,181 DEBUG [RS:0;b70d29849faf:41471 {}] zookeeper.ZKUtil(111): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b70d29849faf,41471,1731369291585 2024-11-11T23:54:54,181 WARN [RS:0;b70d29849faf:41471 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T23:54:54,182 INFO [RS:0;b70d29849faf:41471 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T23:54:54,182 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/WALs/b70d29849faf,41471,1731369291585 2024-11-11T23:54:54,184 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b70d29849faf,41471,1731369291585] 2024-11-11T23:54:54,210 INFO [RS:0;b70d29849faf:41471 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T23:54:54,227 INFO [RS:0;b70d29849faf:41471 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T23:54:54,233 INFO [RS:0;b70d29849faf:41471 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T23:54:54,233 INFO [RS:0;b70d29849faf:41471 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T23:54:54,234 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T23:54:54,239 INFO [RS:0;b70d29849faf:41471 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T23:54:54,240 INFO [RS:0;b70d29849faf:41471 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T23:54:54,241 DEBUG [RS:0;b70d29849faf:41471 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:54:54,241 DEBUG [RS:0;b70d29849faf:41471 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:54:54,241 DEBUG [RS:0;b70d29849faf:41471 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:54:54,241 DEBUG [RS:0;b70d29849faf:41471 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:54:54,242 DEBUG [RS:0;b70d29849faf:41471 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:54:54,242 DEBUG [RS:0;b70d29849faf:41471 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b70d29849faf:0, corePoolSize=2, maxPoolSize=2 2024-11-11T23:54:54,242 DEBUG [RS:0;b70d29849faf:41471 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:54:54,242 DEBUG [RS:0;b70d29849faf:41471 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:54:54,242 DEBUG [RS:0;b70d29849faf:41471 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:54:54,242 DEBUG [RS:0;b70d29849faf:41471 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:54:54,243 DEBUG [RS:0;b70d29849faf:41471 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:54:54,243 DEBUG [RS:0;b70d29849faf:41471 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b70d29849faf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T23:54:54,243 DEBUG [RS:0;b70d29849faf:41471 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b70d29849faf:0, corePoolSize=3, maxPoolSize=3 2024-11-11T23:54:54,243 DEBUG [RS:0;b70d29849faf:41471 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b70d29849faf:0, corePoolSize=3, maxPoolSize=3 2024-11-11T23:54:54,245 INFO [RS:0;b70d29849faf:41471 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T23:54:54,245 INFO [RS:0;b70d29849faf:41471 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T23:54:54,245 INFO [RS:0;b70d29849faf:41471 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T23:54:54,246 INFO [RS:0;b70d29849faf:41471 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T23:54:54,246 INFO [RS:0;b70d29849faf:41471 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T23:54:54,246 INFO [RS:0;b70d29849faf:41471 {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,41471,1731369291585-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T23:54:54,265 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T23:54:54,267 INFO [RS:0;b70d29849faf:41471 {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,41471,1731369291585-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T23:54:54,267 INFO [RS:0;b70d29849faf:41471 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T23:54:54,267 INFO [RS:0;b70d29849faf:41471 {}] regionserver.Replication(171): b70d29849faf,41471,1731369291585 started 2024-11-11T23:54:54,285 INFO [RS:0;b70d29849faf:41471 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T23:54:54,285 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1482): Serving as b70d29849faf,41471,1731369291585, RpcServer on b70d29849faf/172.17.0.2:41471, sessionid=0x1003136265b0001 2024-11-11T23:54:54,286 DEBUG [RS:0;b70d29849faf:41471 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T23:54:54,286 DEBUG [RS:0;b70d29849faf:41471 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b70d29849faf,41471,1731369291585 2024-11-11T23:54:54,286 DEBUG [RS:0;b70d29849faf:41471 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b70d29849faf,41471,1731369291585' 2024-11-11T23:54:54,286 DEBUG [RS:0;b70d29849faf:41471 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T23:54:54,287 DEBUG [RS:0;b70d29849faf:41471 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T23:54:54,288 DEBUG [RS:0;b70d29849faf:41471 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T23:54:54,288 DEBUG [RS:0;b70d29849faf:41471 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T23:54:54,288 DEBUG [RS:0;b70d29849faf:41471 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b70d29849faf,41471,1731369291585 2024-11-11T23:54:54,288 DEBUG [RS:0;b70d29849faf:41471 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b70d29849faf,41471,1731369291585' 2024-11-11T23:54:54,288 DEBUG [RS:0;b70d29849faf:41471 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T23:54:54,289 DEBUG [RS:0;b70d29849faf:41471 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T23:54:54,290 DEBUG [RS:0;b70d29849faf:41471 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T23:54:54,290 INFO [RS:0;b70d29849faf:41471 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T23:54:54,290 INFO [RS:0;b70d29849faf:41471 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T23:54:54,534 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T23:54:54,536 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018 2024-11-11T23:54:54,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741832_1008 (size=32) 2024-11-11T23:54:54,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:54:54,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T23:54:54,959 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T23:54:54,959 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:54,960 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:54:54,960 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T23:54:54,962 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T23:54:54,962 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:54,963 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:54:54,963 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T23:54:54,966 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T23:54:54,966 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:54,967 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:54:54,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T23:54:54,970 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T23:54:54,971 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:54:54,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:54:54,972 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T23:54:54,974 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740 2024-11-11T23:54:54,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740 2024-11-11T23:54:54,978 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T23:54:54,978 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T23:54:54,979 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T23:54:54,981 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T23:54:54,985 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:54:54,986 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68930502, jitterRate=0.027144521474838257}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T23:54:54,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731369294953Initializing all the Stores at 1731369294955 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731369294955Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731369294956 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369294956Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731369294956Cleaning up temporary data from old regions at 1731369294978 (+22 ms)Region opened successfully at 1731369294988 (+10 ms) 2024-11-11T23:54:54,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T23:54:54,988 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T23:54:54,989 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T23:54:54,989 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T23:54:54,989 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T23:54:54,990 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T23:54:54,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731369294988Disabling compacts and flushes for region at 1731369294988Disabling writes for close at 1731369294989 (+1 ms)Writing region close event to WAL at 1731369294990 (+1 ms)Closed at 1731369294990 2024-11-11T23:54:54,993 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T23:54:54,993 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T23:54:54,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T23:54:55,006 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T23:54:55,009 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T23:54:55,161 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:54:56,162 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:54:57,164 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:54:58,165 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:54:59,166 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:00,166 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:01,167 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:01,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T23:55:01,343 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T23:55:02,168 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:03,168 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:04,169 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:05,170 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:06,170 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:07,171 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:08,171 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:09,172 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:10,173 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:11,173 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:12,174 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:13,175 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:14,175 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:15,176 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:16,177 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:17,177 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:18,178 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:19,179 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:20,164 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T23:55:20,180 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:21,181 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:22,182 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:23,183 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:24,184 WARN [b70d29849faf:34765 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T23:55:24,302 INFO [RS:0;b70d29849faf:41471 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T23:55:24,306 INFO [RS:0;b70d29849faf:41471 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b70d29849faf%2C41471%2C1731369291585, suffix=, logDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/WALs/b70d29849faf,41471,1731369291585, archiveDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/oldWALs, maxLogs=32 2024-11-11T23:55:24,320 DEBUG [RS:0;b70d29849faf:41471 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/WALs/b70d29849faf,41471,1731369291585/b70d29849faf%2C41471%2C1731369291585.1731369324308, exclude list is [], retry=0 2024-11-11T23:55:24,325 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37981,DS-640312f5-10b0-40ed-9882-7da92da622b1,DISK] 2024-11-11T23:55:24,328 INFO [RS:0;b70d29849faf:41471 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/WALs/b70d29849faf,41471,1731369291585/b70d29849faf%2C41471%2C1731369291585.1731369324308 2024-11-11T23:55:24,328 DEBUG [RS:0;b70d29849faf:41471 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42225:42225)] 2024-11-11T23:55:24,437 DEBUG [b70d29849faf:34765 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T23:55:24,449 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:24,455 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b70d29849faf,41471,1731369291585, state=OPENING 2024-11-11T23:55:24,459 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T23:55:24,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:55:24,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:55:24,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T23:55:24,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T23:55:24,463 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T23:55:24,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b70d29849faf,41471,1731369291585}] 2024-11-11T23:55:24,645 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T23:55:24,649 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53647, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T23:55:24,661 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T23:55:24,662 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T23:55:24,662 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T23:55:24,665 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b70d29849faf%2C41471%2C1731369291585.meta, suffix=.meta, logDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/WALs/b70d29849faf,41471,1731369291585, archiveDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/oldWALs, maxLogs=32 2024-11-11T23:55:24,680 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/WALs/b70d29849faf,41471,1731369291585/b70d29849faf%2C41471%2C1731369291585.meta.1731369324667.meta, exclude list is [], retry=0 2024-11-11T23:55:24,684 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37981,DS-640312f5-10b0-40ed-9882-7da92da622b1,DISK] 2024-11-11T23:55:24,687 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/WALs/b70d29849faf,41471,1731369291585/b70d29849faf%2C41471%2C1731369291585.meta.1731369324667.meta 2024-11-11T23:55:24,687 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42225:42225)] 2024-11-11T23:55:24,688 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:55:24,689 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T23:55:24,691 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T23:55:24,695 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T23:55:24,699 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T23:55:24,700 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:24,700 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T23:55:24,700 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T23:55:24,703 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T23:55:24,705 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T23:55:24,705 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:24,706 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:55:24,706 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T23:55:24,708 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T23:55:24,708 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:24,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:55:24,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T23:55:24,710 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T23:55:24,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:24,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:55:24,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T23:55:24,713 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T23:55:24,713 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:24,713 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T23:55:24,714 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T23:55:24,715 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740 2024-11-11T23:55:24,718 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740 2024-11-11T23:55:24,721 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T23:55:24,721 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T23:55:24,722 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T23:55:24,725 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T23:55:24,727 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62799682, jitterRate=-0.06421181559562683}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T23:55:24,727 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T23:55:24,729 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731369324701Writing region info on filesystem at 1731369324701Initializing all the Stores at 1731369324703 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731369324703Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731369324703Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369324703Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731369324703Cleaning up temporary data from old regions at 1731369324721 (+18 ms)Running coprocessor post-open hooks at 1731369324727 (+6 ms)Region opened successfully at 1731369324728 (+1 ms) 2024-11-11T23:55:24,735 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731369324635 2024-11-11T23:55:24,746 DEBUG [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T23:55:24,746 INFO [RS_OPEN_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T23:55:24,748 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:24,750 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b70d29849faf,41471,1731369291585, state=OPEN 2024-11-11T23:55:24,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T23:55:24,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T23:55:24,753 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T23:55:24,753 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T23:55:24,754 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b70d29849faf,41471,1731369291585 2024-11-11T23:55:24,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T23:55:24,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b70d29849faf,41471,1731369291585 in 289 msec 2024-11-11T23:55:24,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T23:55:24,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.7640 sec 2024-11-11T23:55:24,769 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T23:55:24,769 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T23:55:24,786 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:24,787 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,41471,1731369291585, seqNum=-1] 2024-11-11T23:55:24,808 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:24,811 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37861, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:24,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.8000 sec 2024-11-11T23:55:24,830 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731369324830, completionTime=-1 2024-11-11T23:55:24,834 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T23:55:24,834 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T23:55:24,863 INFO [master/b70d29849faf:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-11T23:55:24,863 INFO [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731369384863 2024-11-11T23:55:24,863 INFO [master/b70d29849faf:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731369444863 2024-11-11T23:55:24,864 INFO [master/b70d29849faf:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-11T23:55:24,865 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,34765,1731369291015-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T23:55:24,865 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,34765,1731369291015-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T23:55:24,866 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,34765,1731369291015-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T23:55:24,867 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b70d29849faf:34765, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T23:55:24,867 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T23:55:24,868 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T23:55:24,873 DEBUG [master/b70d29849faf:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T23:55:24,894 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.177sec 2024-11-11T23:55:24,895 INFO [master/b70d29849faf:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T23:55:24,896 INFO [master/b70d29849faf:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T23:55:24,897 INFO [master/b70d29849faf:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T23:55:24,898 INFO [master/b70d29849faf:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T23:55:24,898 INFO [master/b70d29849faf:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T23:55:24,899 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,34765,1731369291015-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T23:55:24,900 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,34765,1731369291015-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T23:55:24,907 DEBUG [master/b70d29849faf:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T23:55:24,908 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T23:55:24,908 INFO [master/b70d29849faf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b70d29849faf,34765,1731369291015-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T23:55:24,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573d5973, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:24,918 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T23:55:24,918 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T23:55:24,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,34765,-1 for getting cluster id 2024-11-11T23:55:24,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:24,931 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0c7838f7-a8db-4c18-9e8a-3281aa44d78f' 2024-11-11T23:55:24,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:24,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0c7838f7-a8db-4c18-9e8a-3281aa44d78f" 2024-11-11T23:55:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aafede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,34765,-1] 2024-11-11T23:55:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:24,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:24,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:24,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1928876b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:24,944 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,41471,1731369291585, seqNum=-1] 2024-11-11T23:55:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:24,954 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:24,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b70d29849faf,34765,1731369291015 2024-11-11T23:55:24,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2246aef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:24,973 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b70d29849faf,34765,-1 for getting cluster id 2024-11-11T23:55:24,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T23:55:24,975 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0c7838f7-a8db-4c18-9e8a-3281aa44d78f' 2024-11-11T23:55:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T23:55:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0c7838f7-a8db-4c18-9e8a-3281aa44d78f" 2024-11-11T23:55:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f28ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b70d29849faf,34765,-1] 2024-11-11T23:55:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T23:55:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:55:24,980 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T23:55:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64026d86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T23:55:24,994 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=215, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=385, ProcessCount=11, AvailableMemoryMB=4543 2024-11-11T23:55:25,001 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T23:55:25,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b70d29849faf,34765,1731369291015 2024-11-11T23:55:25,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@25463b12 2024-11-11T23:55:25,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T23:55:25,012 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T23:55:25,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:55:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-11T23:55:25,028 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:55:25,030 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-11T23:55:25,030 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:25,033 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:55:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T23:55:25,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741835_1011 (size=428) 2024-11-11T23:55:25,469 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4ea1681b2f6bdcab4ccb742ef1cfbcde, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018 2024-11-11T23:55:25,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741836_1012 (size=79) 2024-11-11T23:55:25,484 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:25,484 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 4ea1681b2f6bdcab4ccb742ef1cfbcde, disabling compactions & flushes 2024-11-11T23:55:25,484 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. 2024-11-11T23:55:25,484 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. 2024-11-11T23:55:25,484 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. after waiting 0 ms 2024-11-11T23:55:25,484 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. 2024-11-11T23:55:25,484 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. 2024-11-11T23:55:25,484 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4ea1681b2f6bdcab4ccb742ef1cfbcde: Waiting for close lock at 1731369325484Disabling compacts and flushes for region at 1731369325484Disabling writes for close at 1731369325484Writing region close event to WAL at 1731369325484Closed at 1731369325484 2024-11-11T23:55:25,487 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:55:25,493 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731369325487"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369325487"}]},"ts":"1731369325487"} 2024-11-11T23:55:25,506 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:55:25,509 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:55:25,512 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369325509"}]},"ts":"1731369325509"} 2024-11-11T23:55:25,518 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-11T23:55:25,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4ea1681b2f6bdcab4ccb742ef1cfbcde, ASSIGN}] 2024-11-11T23:55:25,524 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4ea1681b2f6bdcab4ccb742ef1cfbcde, ASSIGN 2024-11-11T23:55:25,527 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4ea1681b2f6bdcab4ccb742ef1cfbcde, ASSIGN; state=OFFLINE, location=b70d29849faf,41471,1731369291585; forceNewPlan=false, retain=false 2024-11-11T23:55:25,678 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4ea1681b2f6bdcab4ccb742ef1cfbcde, regionState=OPENING, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:25,684 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4ea1681b2f6bdcab4ccb742ef1cfbcde, ASSIGN because future has completed 2024-11-11T23:55:25,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ea1681b2f6bdcab4ccb742ef1cfbcde, server=b70d29849faf,41471,1731369291585}] 2024-11-11T23:55:25,852 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. 2024-11-11T23:55:25,853 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4ea1681b2f6bdcab4ccb742ef1cfbcde, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:55:25,853 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 4ea1681b2f6bdcab4ccb742ef1cfbcde 2024-11-11T23:55:25,853 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:25,853 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4ea1681b2f6bdcab4ccb742ef1cfbcde 2024-11-11T23:55:25,853 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4ea1681b2f6bdcab4ccb742ef1cfbcde 2024-11-11T23:55:25,855 INFO [StoreOpener-4ea1681b2f6bdcab4ccb742ef1cfbcde-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4ea1681b2f6bdcab4ccb742ef1cfbcde 2024-11-11T23:55:25,858 INFO [StoreOpener-4ea1681b2f6bdcab4ccb742ef1cfbcde-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ea1681b2f6bdcab4ccb742ef1cfbcde columnFamilyName testFamily 2024-11-11T23:55:25,858 DEBUG [StoreOpener-4ea1681b2f6bdcab4ccb742ef1cfbcde-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:25,859 INFO [StoreOpener-4ea1681b2f6bdcab4ccb742ef1cfbcde-1 {}] regionserver.HStore(327): Store=4ea1681b2f6bdcab4ccb742ef1cfbcde/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:25,860 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4ea1681b2f6bdcab4ccb742ef1cfbcde 2024-11-11T23:55:25,861 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4ea1681b2f6bdcab4ccb742ef1cfbcde 2024-11-11T23:55:25,861 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4ea1681b2f6bdcab4ccb742ef1cfbcde 2024-11-11T23:55:25,862 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4ea1681b2f6bdcab4ccb742ef1cfbcde 2024-11-11T23:55:25,862 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4ea1681b2f6bdcab4ccb742ef1cfbcde 2024-11-11T23:55:25,865 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4ea1681b2f6bdcab4ccb742ef1cfbcde 2024-11-11T23:55:25,870 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4ea1681b2f6bdcab4ccb742ef1cfbcde/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:55:25,871 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4ea1681b2f6bdcab4ccb742ef1cfbcde; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61779101, jitterRate=-0.07941965758800507}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T23:55:25,871 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4ea1681b2f6bdcab4ccb742ef1cfbcde 2024-11-11T23:55:25,872 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4ea1681b2f6bdcab4ccb742ef1cfbcde: Running coprocessor pre-open hook at 1731369325853Writing region info on filesystem at 1731369325853Initializing all the Stores at 1731369325855 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369325855Cleaning up temporary data from old regions at 1731369325862 (+7 ms)Running coprocessor post-open hooks at 1731369325871 (+9 ms)Region opened successfully at 1731369325872 (+1 ms) 2024-11-11T23:55:25,874 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde., pid=6, masterSystemTime=1731369325845 2024-11-11T23:55:25,879 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. 2024-11-11T23:55:25,879 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. 2024-11-11T23:55:25,879 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4ea1681b2f6bdcab4ccb742ef1cfbcde, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:25,885 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ea1681b2f6bdcab4ccb742ef1cfbcde, server=b70d29849faf,41471,1731369291585 because future has completed 2024-11-11T23:55:25,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T23:55:25,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4ea1681b2f6bdcab4ccb742ef1cfbcde, server=b70d29849faf,41471,1731369291585 in 199 msec 2024-11-11T23:55:25,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T23:55:25,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4ea1681b2f6bdcab4ccb742ef1cfbcde, ASSIGN in 375 msec 2024-11-11T23:55:25,903 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:55:25,903 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369325903"}]},"ts":"1731369325903"} 2024-11-11T23:55:25,909 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-11T23:55:25,911 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:55:25,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 890 msec 2024-11-11T23:55:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T23:55:26,070 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-11T23:55:26,070 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-11T23:55:26,071 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:55:26,077 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-11T23:55:26,077 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:55:26,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-11T23:55:26,080 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T23:55:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b70d29849faf,41471,1731369291585, seqNum=-1] 2024-11-11T23:55:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T23:55:26,085 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T23:55:26,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde., hostname=b70d29849faf,41471,1731369291585, seqNum=2] 2024-11-11T23:55:26,120 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-11T23:55:26,123 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-11T23:55:26,129 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-11T23:55:26,139 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-11T23:55:26,141 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-11T23:55:26,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-2070106616706891768 2024-11-11T23:55:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:45872 deadline: 1731369329141, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-11T23:55:26,180 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-11T23:55:26,182 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-11T23:55:26,183 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-11T23:55:26,185 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-11T23:55:26,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-2070106616706891766 2024-11-11T23:55:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:45872 deadline: 1731369329185, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-11T23:55:26,193 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-11T23:55:26,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-2070106616706891765 2024-11-11T23:55:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:45872 deadline: 1731369329192, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-11T23:55:26,198 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-11T23:55:26,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-2070106616706891764 2024-11-11T23:55:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:45872 deadline: 1731369329198, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-11T23:55:26,205 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-11T23:55:26,228 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=221 (was 215) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1202997173_22 at /127.0.0.1:47646 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/b70d29849faf:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=385 (was 385), ProcessCount=11 (was 11), AvailableMemoryMB=4488 (was 4543) 2024-11-11T23:55:26,245 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=221, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=385, ProcessCount=11, AvailableMemoryMB=4486 2024-11-11T23:55:26,249 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34765 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:55:26,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34765 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-11T23:55:26,254 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:55:26,255 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:26,255 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34765 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-11T23:55:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T23:55:26,257 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:55:26,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741837_1013 (size=414) 2024-11-11T23:55:26,676 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f42f7a1136d0e434e5b2c914b87b4653, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018 2024-11-11T23:55:26,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741838_1014 (size=65) 2024-11-11T23:55:27,086 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:27,086 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing f42f7a1136d0e434e5b2c914b87b4653, disabling compactions & flushes 2024-11-11T23:55:27,086 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. 2024-11-11T23:55:27,086 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. 2024-11-11T23:55:27,086 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. after waiting 0 ms 2024-11-11T23:55:27,086 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. 2024-11-11T23:55:27,086 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. 2024-11-11T23:55:27,086 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for f42f7a1136d0e434e5b2c914b87b4653: Waiting for close lock at 1731369327086Disabling compacts and flushes for region at 1731369327086Disabling writes for close at 1731369327086Writing region close event to WAL at 1731369327086Closed at 1731369327086 2024-11-11T23:55:27,089 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:55:27,089 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1731369327089"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369327089"}]},"ts":"1731369327089"} 2024-11-11T23:55:27,094 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:55:27,097 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:55:27,098 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369327097"}]},"ts":"1731369327097"} 2024-11-11T23:55:27,101 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-11T23:55:27,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=f42f7a1136d0e434e5b2c914b87b4653, ASSIGN}] 2024-11-11T23:55:27,104 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=f42f7a1136d0e434e5b2c914b87b4653, ASSIGN 2024-11-11T23:55:27,107 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=f42f7a1136d0e434e5b2c914b87b4653, ASSIGN; state=OFFLINE, location=b70d29849faf,41471,1731369291585; forceNewPlan=false, retain=false 2024-11-11T23:55:27,258 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=f42f7a1136d0e434e5b2c914b87b4653, regionState=OPENING, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:27,262 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=f42f7a1136d0e434e5b2c914b87b4653, ASSIGN because future has completed 2024-11-11T23:55:27,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure f42f7a1136d0e434e5b2c914b87b4653, server=b70d29849faf,41471,1731369291585}] 2024-11-11T23:55:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T23:55:27,427 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. 2024-11-11T23:55:27,428 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => f42f7a1136d0e434e5b2c914b87b4653, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:55:27,428 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner f42f7a1136d0e434e5b2c914b87b4653 2024-11-11T23:55:27,428 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:27,428 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for f42f7a1136d0e434e5b2c914b87b4653 2024-11-11T23:55:27,428 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for f42f7a1136d0e434e5b2c914b87b4653 2024-11-11T23:55:27,431 INFO [StoreOpener-f42f7a1136d0e434e5b2c914b87b4653-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f42f7a1136d0e434e5b2c914b87b4653 2024-11-11T23:55:27,433 INFO [StoreOpener-f42f7a1136d0e434e5b2c914b87b4653-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f42f7a1136d0e434e5b2c914b87b4653 columnFamilyName testFamily 2024-11-11T23:55:27,433 DEBUG [StoreOpener-f42f7a1136d0e434e5b2c914b87b4653-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:27,434 INFO [StoreOpener-f42f7a1136d0e434e5b2c914b87b4653-1 {}] regionserver.HStore(327): Store=f42f7a1136d0e434e5b2c914b87b4653/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:27,434 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for f42f7a1136d0e434e5b2c914b87b4653 2024-11-11T23:55:27,436 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScanner/f42f7a1136d0e434e5b2c914b87b4653 2024-11-11T23:55:27,436 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScanner/f42f7a1136d0e434e5b2c914b87b4653 2024-11-11T23:55:27,437 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for f42f7a1136d0e434e5b2c914b87b4653 2024-11-11T23:55:27,437 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for f42f7a1136d0e434e5b2c914b87b4653 2024-11-11T23:55:27,440 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for f42f7a1136d0e434e5b2c914b87b4653 2024-11-11T23:55:27,444 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScanner/f42f7a1136d0e434e5b2c914b87b4653/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:55:27,445 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened f42f7a1136d0e434e5b2c914b87b4653; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73074626, jitterRate=0.08889678120613098}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T23:55:27,445 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f42f7a1136d0e434e5b2c914b87b4653 2024-11-11T23:55:27,447 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for f42f7a1136d0e434e5b2c914b87b4653: Running coprocessor pre-open hook at 1731369327428Writing region info on filesystem at 1731369327428Initializing all the Stores at 1731369327430 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369327430Cleaning up temporary data from old regions at 1731369327437 (+7 ms)Running coprocessor post-open hooks at 1731369327445 (+8 ms)Region opened successfully at 1731369327446 (+1 ms) 2024-11-11T23:55:27,448 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653., pid=9, masterSystemTime=1731369327420 2024-11-11T23:55:27,454 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. 2024-11-11T23:55:27,454 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. 2024-11-11T23:55:27,454 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=f42f7a1136d0e434e5b2c914b87b4653, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:27,458 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure f42f7a1136d0e434e5b2c914b87b4653, server=b70d29849faf,41471,1731369291585 because future has completed 2024-11-11T23:55:27,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-11T23:55:27,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure f42f7a1136d0e434e5b2c914b87b4653, server=b70d29849faf,41471,1731369291585 in 196 msec 2024-11-11T23:55:27,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T23:55:27,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=f42f7a1136d0e434e5b2c914b87b4653, ASSIGN in 362 msec 2024-11-11T23:55:27,471 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:55:27,471 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369327471"}]},"ts":"1731369327471"} 2024-11-11T23:55:27,477 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-11T23:55:27,479 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:55:27,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2300 sec 2024-11-11T23:55:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T23:55:29,278 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-11T23:55:29,278 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-11T23:55:29,278 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:55:29,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-11T23:55:29,284 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:55:29,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-11T23:55:29,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653., hostname=b70d29849faf,41471,1731369291585, seqNum=2] 2024-11-11T23:55:29,304 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-11T23:55:29,304 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-11T23:55:29,309 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-11T23:55:29,311 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-11T23:55:30,248 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:55:30,283 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-11T23:55:30,285 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T23:55:30,286 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-11T23:55:31,342 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T23:55:31,342 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T23:55:31,344 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T23:55:31,344 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T23:55:31,344 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-11T23:55:31,344 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-11T23:55:31,345 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-11T23:55:31,346 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-11T23:55:35,324 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653., hostname=b70d29849faf,41471,1731369291585, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653., hostname=b70d29849faf,41471,1731369291585, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=b70d29849faf:41471 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6011ms, rpcTimeout=5999ms 2024-11-11T23:55:35,324 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653., hostname=b70d29849faf,41471,1731369291585, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6011ms, rpcTimeout=5999ms 2024-11-11T23:55:35,325 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653., hostname=b70d29849faf,41471,1731369291585, seqNum=2 from cache 2024-11-11T23:55:35,334 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=224 (was 221) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=396 (was 385) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4474 (was 4486) 2024-11-11T23:55:35,342 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=224, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=396, ProcessCount=11, AvailableMemoryMB=4473 2024-11-11T23:55:35,345 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34765 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:55:35,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34765 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-11T23:55:35,350 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:55:35,350 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:35,350 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34765 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-11T23:55:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-11T23:55:35,352 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:55:35,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741839_1015 (size=412) 2024-11-11T23:55:35,764 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 981c9fbb3e7f4c993ecad2d02d339d83, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018 2024-11-11T23:55:35,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741840_1016 (size=63) 2024-11-11T23:55:35,775 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:35,775 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 981c9fbb3e7f4c993ecad2d02d339d83, disabling compactions & flushes 2024-11-11T23:55:35,775 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. 2024-11-11T23:55:35,775 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. 2024-11-11T23:55:35,775 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. after waiting 0 ms 2024-11-11T23:55:35,775 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. 2024-11-11T23:55:35,776 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. 2024-11-11T23:55:35,776 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 981c9fbb3e7f4c993ecad2d02d339d83: Waiting for close lock at 1731369335775Disabling compacts and flushes for region at 1731369335775Disabling writes for close at 1731369335775Writing region close event to WAL at 1731369335776 (+1 ms)Closed at 1731369335776 2024-11-11T23:55:35,778 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:55:35,779 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1731369335779"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369335779"}]},"ts":"1731369335779"} 2024-11-11T23:55:35,784 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:55:35,786 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:55:35,787 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369335786"}]},"ts":"1731369335786"} 2024-11-11T23:55:35,790 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-11T23:55:35,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=981c9fbb3e7f4c993ecad2d02d339d83, ASSIGN}] 2024-11-11T23:55:35,793 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=981c9fbb3e7f4c993ecad2d02d339d83, ASSIGN 2024-11-11T23:55:35,794 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=981c9fbb3e7f4c993ecad2d02d339d83, ASSIGN; state=OFFLINE, location=b70d29849faf,41471,1731369291585; forceNewPlan=false, retain=false 2024-11-11T23:55:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-11T23:55:35,945 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=981c9fbb3e7f4c993ecad2d02d339d83, regionState=OPENING, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:35,951 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=981c9fbb3e7f4c993ecad2d02d339d83, ASSIGN because future has completed 2024-11-11T23:55:35,952 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 981c9fbb3e7f4c993ecad2d02d339d83, server=b70d29849faf,41471,1731369291585}] 2024-11-11T23:55:36,114 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. 2024-11-11T23:55:36,115 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 981c9fbb3e7f4c993ecad2d02d339d83, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:55:36,115 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:55:36,115 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:36,115 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:55:36,115 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:55:36,118 INFO [StoreOpener-981c9fbb3e7f4c993ecad2d02d339d83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:55:36,120 INFO [StoreOpener-981c9fbb3e7f4c993ecad2d02d339d83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 981c9fbb3e7f4c993ecad2d02d339d83 columnFamilyName testFamily 2024-11-11T23:55:36,120 DEBUG [StoreOpener-981c9fbb3e7f4c993ecad2d02d339d83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:36,120 INFO [StoreOpener-981c9fbb3e7f4c993ecad2d02d339d83-1 {}] regionserver.HStore(327): Store=981c9fbb3e7f4c993ecad2d02d339d83/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:36,121 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:55:36,122 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNextAsync/981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:55:36,122 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNextAsync/981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:55:36,123 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:55:36,123 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:55:36,126 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:55:36,129 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNextAsync/981c9fbb3e7f4c993ecad2d02d339d83/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:55:36,130 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 981c9fbb3e7f4c993ecad2d02d339d83; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67120940, jitterRate=1.799464225769043E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T23:55:36,130 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:55:36,131 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 981c9fbb3e7f4c993ecad2d02d339d83: Running coprocessor pre-open hook at 1731369336116Writing region info on filesystem at 1731369336116Initializing all the Stores at 1731369336117 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369336117Cleaning up temporary data from old regions at 1731369336123 (+6 ms)Running coprocessor post-open hooks at 1731369336130 (+7 ms)Region opened successfully at 1731369336131 (+1 ms) 2024-11-11T23:55:36,132 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83., pid=12, masterSystemTime=1731369336106 2024-11-11T23:55:36,136 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. 2024-11-11T23:55:36,136 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. 2024-11-11T23:55:36,137 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=981c9fbb3e7f4c993ecad2d02d339d83, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:36,141 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 981c9fbb3e7f4c993ecad2d02d339d83, server=b70d29849faf,41471,1731369291585 because future has completed 2024-11-11T23:55:36,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T23:55:36,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 981c9fbb3e7f4c993ecad2d02d339d83, server=b70d29849faf,41471,1731369291585 in 191 msec 2024-11-11T23:55:36,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-11T23:55:36,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=981c9fbb3e7f4c993ecad2d02d339d83, ASSIGN in 357 msec 2024-11-11T23:55:36,153 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:55:36,153 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369336153"}]},"ts":"1731369336153"} 2024-11-11T23:55:36,157 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-11T23:55:36,159 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:55:36,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 813 msec 2024-11-11T23:55:36,287 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-11T23:55:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-11T23:55:36,368 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-11T23:55:36,368 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-11T23:55:36,369 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:55:36,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-11T23:55:36,374 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:55:36,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-11T23:55:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83., hostname=b70d29849faf,41471,1731369291585, seqNum=2] 2024-11-11T23:55:36,392 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-11T23:55:36,392 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-11T23:55:36,396 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-11T23:55:36,400 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-11T23:55:36,401 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-11T23:55:36,402 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-11T23:55:41,647 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:55:41,682 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-11T23:55:41,682 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-11T23:55:42,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41471 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2070106616706891755' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-11T23:55:42,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41471 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:45872 deadline: 1731369348409, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2070106616706891755' does not exist 2024-11-11T23:55:42,413 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to b70d29849faf,41471,1731369291585 for closing scanner id = -2070106616706891755 for 981c9fbb3e7f4c993ecad2d02d339d83 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2070106616706891755' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:42,417 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=226 (was 224) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=404 (was 396) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4455 (was 4473) 2024-11-11T23:55:42,427 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=226, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=404, ProcessCount=11, AvailableMemoryMB=4455 2024-11-11T23:55:42,430 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:55:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-11T23:55:42,433 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:55:42,434 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:42,434 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-11T23:55:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T23:55:42,436 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:55:42,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741841_1017 (size=423) 2024-11-11T23:55:42,848 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 17720cb52cfe4bc507a89874ac61d582, NAME => 'testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018 2024-11-11T23:55:42,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741842_1018 (size=74) 2024-11-11T23:55:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-11T23:55:43,257 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:43,257 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 17720cb52cfe4bc507a89874ac61d582, disabling compactions & flushes 2024-11-11T23:55:43,257 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. 2024-11-11T23:55:43,257 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. 2024-11-11T23:55:43,257 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. after waiting 0 ms 2024-11-11T23:55:43,257 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. 2024-11-11T23:55:43,257 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. 2024-11-11T23:55:43,257 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 17720cb52cfe4bc507a89874ac61d582: Waiting for close lock at 1731369343257Disabling compacts and flushes for region at 1731369343257Disabling writes for close at 1731369343257Writing region close event to WAL at 1731369343257Closed at 1731369343257 2024-11-11T23:55:43,259 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:55:43,260 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731369343259"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369343259"}]},"ts":"1731369343259"} 2024-11-11T23:55:43,263 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:55:43,265 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:55:43,265 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369343265"}]},"ts":"1731369343265"} 2024-11-11T23:55:43,267 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-11T23:55:43,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=17720cb52cfe4bc507a89874ac61d582, ASSIGN}] 2024-11-11T23:55:43,269 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=17720cb52cfe4bc507a89874ac61d582, ASSIGN 2024-11-11T23:55:43,271 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=17720cb52cfe4bc507a89874ac61d582, ASSIGN; state=OFFLINE, location=b70d29849faf,41471,1731369291585; forceNewPlan=false, retain=false 2024-11-11T23:55:43,422 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=17720cb52cfe4bc507a89874ac61d582, regionState=OPENING, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:43,425 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=17720cb52cfe4bc507a89874ac61d582, ASSIGN because future has completed 2024-11-11T23:55:43,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17720cb52cfe4bc507a89874ac61d582, server=b70d29849faf,41471,1731369291585}] 2024-11-11T23:55:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T23:55:43,585 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. 2024-11-11T23:55:43,585 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 17720cb52cfe4bc507a89874ac61d582, NAME => 'testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:55:43,585 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 17720cb52cfe4bc507a89874ac61d582 2024-11-11T23:55:43,585 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:43,586 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 17720cb52cfe4bc507a89874ac61d582 2024-11-11T23:55:43,586 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 17720cb52cfe4bc507a89874ac61d582 2024-11-11T23:55:43,589 INFO [StoreOpener-17720cb52cfe4bc507a89874ac61d582-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 17720cb52cfe4bc507a89874ac61d582 2024-11-11T23:55:43,592 INFO [StoreOpener-17720cb52cfe4bc507a89874ac61d582-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17720cb52cfe4bc507a89874ac61d582 columnFamilyName testFamily 2024-11-11T23:55:43,593 DEBUG [StoreOpener-17720cb52cfe4bc507a89874ac61d582-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:43,594 INFO [StoreOpener-17720cb52cfe4bc507a89874ac61d582-1 {}] regionserver.HStore(327): Store=17720cb52cfe4bc507a89874ac61d582/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:43,594 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 17720cb52cfe4bc507a89874ac61d582 2024-11-11T23:55:43,595 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextException/17720cb52cfe4bc507a89874ac61d582 2024-11-11T23:55:43,596 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextException/17720cb52cfe4bc507a89874ac61d582 2024-11-11T23:55:43,596 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 17720cb52cfe4bc507a89874ac61d582 2024-11-11T23:55:43,596 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 17720cb52cfe4bc507a89874ac61d582 2024-11-11T23:55:43,602 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 17720cb52cfe4bc507a89874ac61d582 2024-11-11T23:55:43,605 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextException/17720cb52cfe4bc507a89874ac61d582/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:55:43,606 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 17720cb52cfe4bc507a89874ac61d582; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66366935, jitterRate=-0.011055603623390198}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T23:55:43,606 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17720cb52cfe4bc507a89874ac61d582 2024-11-11T23:55:43,607 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 17720cb52cfe4bc507a89874ac61d582: Running coprocessor pre-open hook at 1731369343586Writing region info on filesystem at 1731369343586Initializing all the Stores at 1731369343588 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369343588Cleaning up temporary data from old regions at 1731369343596 (+8 ms)Running coprocessor post-open hooks at 1731369343606 (+10 ms)Region opened successfully at 1731369343607 (+1 ms) 2024-11-11T23:55:43,609 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582., pid=15, masterSystemTime=1731369343579 2024-11-11T23:55:43,614 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. 2024-11-11T23:55:43,614 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. 2024-11-11T23:55:43,615 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=17720cb52cfe4bc507a89874ac61d582, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:43,619 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17720cb52cfe4bc507a89874ac61d582, server=b70d29849faf,41471,1731369291585 because future has completed 2024-11-11T23:55:43,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-11T23:55:43,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 17720cb52cfe4bc507a89874ac61d582, server=b70d29849faf,41471,1731369291585 in 195 msec 2024-11-11T23:55:43,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T23:55:43,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=17720cb52cfe4bc507a89874ac61d582, ASSIGN in 358 msec 2024-11-11T23:55:43,632 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:55:43,632 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369343632"}]},"ts":"1731369343632"} 2024-11-11T23:55:43,635 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-11T23:55:43,637 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:55:43,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2070 sec 2024-11-11T23:55:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T23:55:45,458 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-11T23:55:45,458 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-11T23:55:45,459 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:55:45,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-11T23:55:45,463 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:55:45,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-11T23:55:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582., hostname=b70d29849faf,41471,1731369291585, seqNum=2] 2024-11-11T23:55:45,477 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-11T23:55:45,477 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-11T23:55:45,480 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-11T23:55:45,482 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-11T23:55:45,484 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-11T23:55:45,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-2070106616706891751 2024-11-11T23:55:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:45872 deadline: 1731369348484, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-11T23:55:45,489 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-11T23:55:45,490 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-11T23:55:45,491 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-11T23:55:45,493 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-11T23:55:45,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-2070106616706891749 2024-11-11T23:55:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:45872 deadline: 1731369348492, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-11T23:55:45,497 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-11T23:55:45,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-2070106616706891748 2024-11-11T23:55:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:45872 deadline: 1731369348497, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-11T23:55:45,501 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-11T23:55:45,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-2070106616706891747 2024-11-11T23:55:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:45872 deadline: 1731369348501, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-11T23:55:45,505 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-11T23:55:45,513 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=229 (was 226) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1202997173_22 at /127.0.0.1:59300 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=396 (was 404), ProcessCount=11 (was 11), AvailableMemoryMB=4451 (was 4455) 2024-11-11T23:55:45,522 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=229, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=396, ProcessCount=11, AvailableMemoryMB=4450 2024-11-11T23:55:45,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:55:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-11T23:55:45,528 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:55:45,528 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:45,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-11T23:55:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-11T23:55:45,530 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:55:45,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741843_1019 (size=418) 2024-11-11T23:55:45,944 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 65ce0dbfc1952458af49fd41d21f0a5d, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018 2024-11-11T23:55:45,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741844_1020 (size=69) 2024-11-11T23:55:46,358 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:46,358 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 65ce0dbfc1952458af49fd41d21f0a5d, disabling compactions & flushes 2024-11-11T23:55:46,358 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. 2024-11-11T23:55:46,358 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. 2024-11-11T23:55:46,358 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. after waiting 0 ms 2024-11-11T23:55:46,358 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. 2024-11-11T23:55:46,359 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. 2024-11-11T23:55:46,359 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 65ce0dbfc1952458af49fd41d21f0a5d: Waiting for close lock at 1731369346358Disabling compacts and flushes for region at 1731369346358Disabling writes for close at 1731369346358Writing region close event to WAL at 1731369346359 (+1 ms)Closed at 1731369346359 2024-11-11T23:55:46,361 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:55:46,362 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1731369346362"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369346362"}]},"ts":"1731369346362"} 2024-11-11T23:55:46,367 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:55:46,369 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:55:46,369 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369346369"}]},"ts":"1731369346369"} 2024-11-11T23:55:46,372 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-11T23:55:46,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=65ce0dbfc1952458af49fd41d21f0a5d, ASSIGN}] 2024-11-11T23:55:46,374 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=65ce0dbfc1952458af49fd41d21f0a5d, ASSIGN 2024-11-11T23:55:46,375 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=65ce0dbfc1952458af49fd41d21f0a5d, ASSIGN; state=OFFLINE, location=b70d29849faf,41471,1731369291585; forceNewPlan=false, retain=false 2024-11-11T23:55:46,526 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=65ce0dbfc1952458af49fd41d21f0a5d, regionState=OPENING, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:46,532 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=65ce0dbfc1952458af49fd41d21f0a5d, ASSIGN because future has completed 2024-11-11T23:55:46,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65ce0dbfc1952458af49fd41d21f0a5d, server=b70d29849faf,41471,1731369291585}] 2024-11-11T23:55:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-11T23:55:46,697 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. 2024-11-11T23:55:46,697 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 65ce0dbfc1952458af49fd41d21f0a5d, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:55:46,698 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 65ce0dbfc1952458af49fd41d21f0a5d 2024-11-11T23:55:46,698 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:46,698 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 65ce0dbfc1952458af49fd41d21f0a5d 2024-11-11T23:55:46,698 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 65ce0dbfc1952458af49fd41d21f0a5d 2024-11-11T23:55:46,700 INFO [StoreOpener-65ce0dbfc1952458af49fd41d21f0a5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 65ce0dbfc1952458af49fd41d21f0a5d 2024-11-11T23:55:46,701 INFO [StoreOpener-65ce0dbfc1952458af49fd41d21f0a5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65ce0dbfc1952458af49fd41d21f0a5d columnFamilyName testFamily 2024-11-11T23:55:46,702 DEBUG [StoreOpener-65ce0dbfc1952458af49fd41d21f0a5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:46,702 INFO [StoreOpener-65ce0dbfc1952458af49fd41d21f0a5d-1 {}] regionserver.HStore(327): Store=65ce0dbfc1952458af49fd41d21f0a5d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:46,702 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 65ce0dbfc1952458af49fd41d21f0a5d 2024-11-11T23:55:46,703 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScanner/65ce0dbfc1952458af49fd41d21f0a5d 2024-11-11T23:55:46,704 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScanner/65ce0dbfc1952458af49fd41d21f0a5d 2024-11-11T23:55:46,704 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 65ce0dbfc1952458af49fd41d21f0a5d 2024-11-11T23:55:46,704 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 65ce0dbfc1952458af49fd41d21f0a5d 2024-11-11T23:55:46,706 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 65ce0dbfc1952458af49fd41d21f0a5d 2024-11-11T23:55:46,710 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScanner/65ce0dbfc1952458af49fd41d21f0a5d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:55:46,710 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 65ce0dbfc1952458af49fd41d21f0a5d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67992667, jitterRate=0.01316969096660614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T23:55:46,711 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 65ce0dbfc1952458af49fd41d21f0a5d 2024-11-11T23:55:46,711 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 65ce0dbfc1952458af49fd41d21f0a5d: Running coprocessor pre-open hook at 1731369346698Writing region info on filesystem at 1731369346698Initializing all the Stores at 1731369346699 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369346699Cleaning up temporary data from old regions at 1731369346704 (+5 ms)Running coprocessor post-open hooks at 1731369346711 (+7 ms)Region opened successfully at 1731369346711 2024-11-11T23:55:46,713 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d., pid=18, masterSystemTime=1731369346687 2024-11-11T23:55:46,715 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. 2024-11-11T23:55:46,715 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. 2024-11-11T23:55:46,716 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=65ce0dbfc1952458af49fd41d21f0a5d, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:46,719 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65ce0dbfc1952458af49fd41d21f0a5d, server=b70d29849faf,41471,1731369291585 because future has completed 2024-11-11T23:55:46,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T23:55:46,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 65ce0dbfc1952458af49fd41d21f0a5d, server=b70d29849faf,41471,1731369291585 in 189 msec 2024-11-11T23:55:46,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-11T23:55:46,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=65ce0dbfc1952458af49fd41d21f0a5d, ASSIGN in 353 msec 2024-11-11T23:55:46,730 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:55:46,730 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369346730"}]},"ts":"1731369346730"} 2024-11-11T23:55:46,733 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-11T23:55:46,734 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:55:46,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2090 sec 2024-11-11T23:55:48,290 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-11T23:55:48,292 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-11T23:55:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-11T23:55:48,571 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-11T23:55:48,571 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-11T23:55:48,572 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:55:48,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-11T23:55:48,582 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:55:48,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-11T23:55:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d., hostname=b70d29849faf,41471,1731369291585, seqNum=2] 2024-11-11T23:55:48,597 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-11T23:55:48,597 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-11T23:55:48,601 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-11T23:55:48,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-11T23:55:49,088 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:55:49,607 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d., hostname=b70d29849faf,41471,1731369291585, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d., hostname=b70d29849faf,41471,1731369291585, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=b70d29849faf:41471 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-11-11T23:55:49,607 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d., hostname=b70d29849faf,41471,1731369291585, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-11-11T23:55:49,607 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d., hostname=b70d29849faf,41471,1731369291585, seqNum=2 from cache 2024-11-11T23:55:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-11T23:55:50,165 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T23:55:50,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d., hostname=b70d29849faf,41471,1731369291585, seqNum=2] 2024-11-11T23:55:50,639 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-11T23:55:51,342 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-11T23:55:51,342 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-11T23:55:51,344 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-11T23:55:51,344 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-11T23:55:51,606 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d., hostname=b70d29849faf,41471,1731369291585, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d., hostname=b70d29849faf,41471,1731369291585, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=b70d29849faf:41471 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=969ms, rpcTimeout=963ms 2024-11-11T23:55:51,606 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d., hostname=b70d29849faf,41471,1731369291585, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=969ms, rpcTimeout=963ms 2024-11-11T23:55:51,606 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d., hostname=b70d29849faf,41471,1731369291585, seqNum=2 from cache 2024-11-11T23:55:51,619 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 229), OpenFileDescriptor=448 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=364 (was 396), ProcessCount=11 (was 11), AvailableMemoryMB=4446 (was 4450) 2024-11-11T23:55:51,630 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=364, ProcessCount=11, AvailableMemoryMB=4446 2024-11-11T23:55:51,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:55:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-11T23:55:51,637 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:55:51,638 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:51,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-11T23:55:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T23:55:51,639 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:55:51,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741845_1021 (size=416) 2024-11-11T23:55:52,055 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 258cf9e310212682e64135fb4ca21e76, NAME => 'testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018 2024-11-11T23:55:52,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741846_1022 (size=67) 2024-11-11T23:55:52,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-11T23:55:52,472 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:52,473 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 258cf9e310212682e64135fb4ca21e76, disabling compactions & flushes 2024-11-11T23:55:52,473 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. 2024-11-11T23:55:52,473 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. 2024-11-11T23:55:52,473 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. after waiting 0 ms 2024-11-11T23:55:52,473 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. 2024-11-11T23:55:52,474 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. 2024-11-11T23:55:52,474 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 258cf9e310212682e64135fb4ca21e76: Waiting for close lock at 1731369352472Disabling compacts and flushes for region at 1731369352473 (+1 ms)Disabling writes for close at 1731369352473Writing region close event to WAL at 1731369352473Closed at 1731369352473 2024-11-11T23:55:52,480 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:55:52,481 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731369352480"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369352480"}]},"ts":"1731369352480"} 2024-11-11T23:55:52,485 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:55:52,486 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:55:52,487 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369352486"}]},"ts":"1731369352486"} 2024-11-11T23:55:52,490 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-11T23:55:52,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=258cf9e310212682e64135fb4ca21e76, ASSIGN}] 2024-11-11T23:55:52,493 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=258cf9e310212682e64135fb4ca21e76, ASSIGN 2024-11-11T23:55:52,494 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=258cf9e310212682e64135fb4ca21e76, ASSIGN; state=OFFLINE, location=b70d29849faf,41471,1731369291585; forceNewPlan=false, retain=false 2024-11-11T23:55:52,645 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=258cf9e310212682e64135fb4ca21e76, regionState=OPENING, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:52,654 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=258cf9e310212682e64135fb4ca21e76, ASSIGN because future has completed 2024-11-11T23:55:52,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 258cf9e310212682e64135fb4ca21e76, server=b70d29849faf,41471,1731369291585}] 2024-11-11T23:55:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T23:55:52,823 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. 2024-11-11T23:55:52,823 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 258cf9e310212682e64135fb4ca21e76, NAME => 'testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:55:52,824 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 258cf9e310212682e64135fb4ca21e76 2024-11-11T23:55:52,824 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:52,824 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 258cf9e310212682e64135fb4ca21e76 2024-11-11T23:55:52,825 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 258cf9e310212682e64135fb4ca21e76 2024-11-11T23:55:52,828 INFO [StoreOpener-258cf9e310212682e64135fb4ca21e76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 258cf9e310212682e64135fb4ca21e76 2024-11-11T23:55:52,830 INFO [StoreOpener-258cf9e310212682e64135fb4ca21e76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 258cf9e310212682e64135fb4ca21e76 columnFamilyName testFamily 2024-11-11T23:55:52,830 DEBUG [StoreOpener-258cf9e310212682e64135fb4ca21e76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:52,831 INFO [StoreOpener-258cf9e310212682e64135fb4ca21e76-1 {}] regionserver.HStore(327): Store=258cf9e310212682e64135fb4ca21e76/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:52,831 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 258cf9e310212682e64135fb4ca21e76 2024-11-11T23:55:52,832 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNextAsync/258cf9e310212682e64135fb4ca21e76 2024-11-11T23:55:52,833 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNextAsync/258cf9e310212682e64135fb4ca21e76 2024-11-11T23:55:52,833 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 258cf9e310212682e64135fb4ca21e76 2024-11-11T23:55:52,833 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 258cf9e310212682e64135fb4ca21e76 2024-11-11T23:55:52,835 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 258cf9e310212682e64135fb4ca21e76 2024-11-11T23:55:52,838 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNextAsync/258cf9e310212682e64135fb4ca21e76/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:55:52,839 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 258cf9e310212682e64135fb4ca21e76; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64806474, jitterRate=-0.034308284521102905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T23:55:52,839 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 258cf9e310212682e64135fb4ca21e76 2024-11-11T23:55:52,840 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 258cf9e310212682e64135fb4ca21e76: Running coprocessor pre-open hook at 1731369352825Writing region info on filesystem at 1731369352825Initializing all the Stores at 1731369352827 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369352827Cleaning up temporary data from old regions at 1731369352833 (+6 ms)Running coprocessor post-open hooks at 1731369352839 (+6 ms)Region opened successfully at 1731369352840 (+1 ms) 2024-11-11T23:55:52,841 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76., pid=21, masterSystemTime=1731369352811 2024-11-11T23:55:52,843 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. 2024-11-11T23:55:52,843 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. 2024-11-11T23:55:52,844 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=258cf9e310212682e64135fb4ca21e76, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:52,847 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 258cf9e310212682e64135fb4ca21e76, server=b70d29849faf,41471,1731369291585 because future has completed 2024-11-11T23:55:52,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-11T23:55:52,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 258cf9e310212682e64135fb4ca21e76, server=b70d29849faf,41471,1731369291585 in 193 msec 2024-11-11T23:55:52,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T23:55:52,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=258cf9e310212682e64135fb4ca21e76, ASSIGN in 361 msec 2024-11-11T23:55:52,855 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:55:52,855 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369352855"}]},"ts":"1731369352855"} 2024-11-11T23:55:52,857 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-11T23:55:52,858 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:55:52,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2260 sec 2024-11-11T23:55:54,294 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-11T23:55:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T23:55:54,678 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-11T23:55:54,678 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-11T23:55:54,678 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:55:54,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-11T23:55:54,684 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:55:54,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-11T23:55:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76., hostname=b70d29849faf,41471,1731369291585, seqNum=2] 2024-11-11T23:55:54,702 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-11T23:55:54,702 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-11T23:55:54,706 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-11T23:55:54,709 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-11T23:55:54,710 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-11T23:55:54,711 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-11T23:55:57,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41471 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2070106616706891736' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-11T23:55:57,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41471 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:45872 deadline: 1731369358719, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2070106616706891736' does not exist 2024-11-11T23:55:57,721 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to b70d29849faf,41471,1731369291585 for closing scanner id = -2070106616706891736 for 258cf9e310212682e64135fb4ca21e76 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2070106616706891736' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:55:57,726 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=446 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 364), ProcessCount=11 (was 11), AvailableMemoryMB=4430 (was 4446) 2024-11-11T23:55:57,736 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=4429 2024-11-11T23:55:57,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:55:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-11T23:55:57,745 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:55:57,745 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:57,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-11T23:55:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-11T23:55:57,747 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:55:57,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741847_1023 (size=411) 2024-11-11T23:55:57,759 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6c54b823914e3bebb023069112e27160, NAME => 'testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018 2024-11-11T23:55:57,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741848_1024 (size=62) 2024-11-11T23:55:57,789 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:57,789 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 6c54b823914e3bebb023069112e27160, disabling compactions & flushes 2024-11-11T23:55:57,789 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. 2024-11-11T23:55:57,789 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. 2024-11-11T23:55:57,789 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. after waiting 0 ms 2024-11-11T23:55:57,789 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. 2024-11-11T23:55:57,789 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. 2024-11-11T23:55:57,790 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6c54b823914e3bebb023069112e27160: Waiting for close lock at 1731369357789Disabling compacts and flushes for region at 1731369357789Disabling writes for close at 1731369357789Writing region close event to WAL at 1731369357789Closed at 1731369357789 2024-11-11T23:55:57,791 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:55:57,792 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731369357792"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369357792"}]},"ts":"1731369357792"} 2024-11-11T23:55:57,796 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:55:57,798 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:55:57,799 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369357798"}]},"ts":"1731369357798"} 2024-11-11T23:55:57,802 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-11T23:55:57,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=6c54b823914e3bebb023069112e27160, ASSIGN}] 2024-11-11T23:55:57,808 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=6c54b823914e3bebb023069112e27160, ASSIGN 2024-11-11T23:55:57,809 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=6c54b823914e3bebb023069112e27160, ASSIGN; state=OFFLINE, location=b70d29849faf,41471,1731369291585; forceNewPlan=false, retain=false 2024-11-11T23:55:57,960 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=6c54b823914e3bebb023069112e27160, regionState=OPENING, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:57,963 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=6c54b823914e3bebb023069112e27160, ASSIGN because future has completed 2024-11-11T23:55:57,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c54b823914e3bebb023069112e27160, server=b70d29849faf,41471,1731369291585}] 2024-11-11T23:55:58,121 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. 2024-11-11T23:55:58,121 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c54b823914e3bebb023069112e27160, NAME => 'testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:55:58,122 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 6c54b823914e3bebb023069112e27160 2024-11-11T23:55:58,122 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:55:58,122 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 6c54b823914e3bebb023069112e27160 2024-11-11T23:55:58,122 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 6c54b823914e3bebb023069112e27160 2024-11-11T23:55:58,124 INFO [StoreOpener-6c54b823914e3bebb023069112e27160-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6c54b823914e3bebb023069112e27160 2024-11-11T23:55:58,126 INFO [StoreOpener-6c54b823914e3bebb023069112e27160-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c54b823914e3bebb023069112e27160 columnFamilyName testFamily 2024-11-11T23:55:58,126 DEBUG [StoreOpener-6c54b823914e3bebb023069112e27160-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:55:58,126 INFO [StoreOpener-6c54b823914e3bebb023069112e27160-1 {}] regionserver.HStore(327): Store=6c54b823914e3bebb023069112e27160/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:55:58,127 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 6c54b823914e3bebb023069112e27160 2024-11-11T23:55:58,128 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNext/6c54b823914e3bebb023069112e27160 2024-11-11T23:55:58,129 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNext/6c54b823914e3bebb023069112e27160 2024-11-11T23:55:58,129 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 6c54b823914e3bebb023069112e27160 2024-11-11T23:55:58,129 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 6c54b823914e3bebb023069112e27160 2024-11-11T23:55:58,132 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 6c54b823914e3bebb023069112e27160 2024-11-11T23:55:58,135 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNext/6c54b823914e3bebb023069112e27160/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:55:58,135 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 6c54b823914e3bebb023069112e27160; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61945178, jitterRate=-0.07694491744041443}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T23:55:58,136 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c54b823914e3bebb023069112e27160 2024-11-11T23:55:58,136 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 6c54b823914e3bebb023069112e27160: Running coprocessor pre-open hook at 1731369358122Writing region info on filesystem at 1731369358122Initializing all the Stores at 1731369358123 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369358123Cleaning up temporary data from old regions at 1731369358129 (+6 ms)Running coprocessor post-open hooks at 1731369358136 (+7 ms)Region opened successfully at 1731369358136 2024-11-11T23:55:58,138 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160., pid=24, masterSystemTime=1731369358116 2024-11-11T23:55:58,140 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. 2024-11-11T23:55:58,141 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. 2024-11-11T23:55:58,141 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=6c54b823914e3bebb023069112e27160, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:55:58,144 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c54b823914e3bebb023069112e27160, server=b70d29849faf,41471,1731369291585 because future has completed 2024-11-11T23:55:58,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-11T23:55:58,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c54b823914e3bebb023069112e27160, server=b70d29849faf,41471,1731369291585 in 183 msec 2024-11-11T23:55:58,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-11T23:55:58,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=6c54b823914e3bebb023069112e27160, ASSIGN in 347 msec 2024-11-11T23:55:58,154 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:55:58,154 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369358154"}]},"ts":"1731369358154"} 2024-11-11T23:55:58,157 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-11T23:55:58,159 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:55:58,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 420 msec 2024-11-11T23:55:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-11T23:55:58,327 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:55:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-11T23:55:58,758 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-11T23:55:58,759 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-11T23:55:58,759 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:55:58,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-11T23:55:58,765 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:55:58,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-11T23:55:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160., hostname=b70d29849faf,41471,1731369291585, seqNum=2] 2024-11-11T23:55:58,782 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-11T23:55:58,782 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-11T23:55:58,786 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-11T23:55:58,789 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-11T23:55:58,790 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-11T23:55:58,791 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-11T23:56:00,296 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-11T23:56:01,342 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-11T23:56:01,342 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-11T23:56:01,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-11T23:56:01,344 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-11T23:56:01,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41471 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2070106616706891732' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-11T23:56:01,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41471 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:45872 deadline: 1731369362800, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2070106616706891732' does not exist 2024-11-11T23:56:01,801 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to b70d29849faf,41471,1731369291585 for closing scanner id = -2070106616706891732 for 6c54b823914e3bebb023069112e27160 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2070106616706891732' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:01,806 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1202997173_22 at /127.0.0.1:58540 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=332 (was 335), ProcessCount=11 (was 11), AvailableMemoryMB=4425 (was 4429) 2024-11-11T23:56:01,815 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=332, ProcessCount=11, AvailableMemoryMB=4424 2024-11-11T23:56:01,818 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34765 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:56:01,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34765 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-11T23:56:01,820 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:56:01,821 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:01,821 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34765 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-11T23:56:01,822 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:56:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T23:56:01,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741849_1025 (size=419) 2024-11-11T23:56:02,233 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f4e97eb825452964401fb63dd561ff82, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018 2024-11-11T23:56:02,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741850_1026 (size=70) 2024-11-11T23:56:02,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-11T23:56:02,643 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:56:02,643 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing f4e97eb825452964401fb63dd561ff82, disabling compactions & flushes 2024-11-11T23:56:02,643 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. 2024-11-11T23:56:02,643 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. 2024-11-11T23:56:02,643 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. after waiting 0 ms 2024-11-11T23:56:02,643 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. 2024-11-11T23:56:02,643 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. 2024-11-11T23:56:02,643 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for f4e97eb825452964401fb63dd561ff82: Waiting for close lock at 1731369362643Disabling compacts and flushes for region at 1731369362643Disabling writes for close at 1731369362643Writing region close event to WAL at 1731369362643Closed at 1731369362643 2024-11-11T23:56:02,645 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:56:02,645 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731369362645"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369362645"}]},"ts":"1731369362645"} 2024-11-11T23:56:02,648 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:56:02,649 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:56:02,650 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369362649"}]},"ts":"1731369362649"} 2024-11-11T23:56:02,652 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-11T23:56:02,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=f4e97eb825452964401fb63dd561ff82, ASSIGN}] 2024-11-11T23:56:02,654 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=f4e97eb825452964401fb63dd561ff82, ASSIGN 2024-11-11T23:56:02,655 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=f4e97eb825452964401fb63dd561ff82, ASSIGN; state=OFFLINE, location=b70d29849faf,41471,1731369291585; forceNewPlan=false, retain=false 2024-11-11T23:56:02,806 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=f4e97eb825452964401fb63dd561ff82, regionState=OPENING, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:56:02,809 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=f4e97eb825452964401fb63dd561ff82, ASSIGN because future has completed 2024-11-11T23:56:02,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4e97eb825452964401fb63dd561ff82, server=b70d29849faf,41471,1731369291585}] 2024-11-11T23:56:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T23:56:02,967 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. 2024-11-11T23:56:02,967 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => f4e97eb825452964401fb63dd561ff82, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:56:02,968 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:02,968 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:56:02,968 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:02,968 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:02,970 INFO [StoreOpener-f4e97eb825452964401fb63dd561ff82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:02,972 INFO [StoreOpener-f4e97eb825452964401fb63dd561ff82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4e97eb825452964401fb63dd561ff82 columnFamilyName testFamily 2024-11-11T23:56:02,973 DEBUG [StoreOpener-f4e97eb825452964401fb63dd561ff82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:02,973 INFO [StoreOpener-f4e97eb825452964401fb63dd561ff82-1 {}] regionserver.HStore(327): Store=f4e97eb825452964401fb63dd561ff82/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:56:02,974 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:02,975 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:02,975 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:02,977 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:02,977 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:02,980 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:02,986 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f4e97eb825452964401fb63dd561ff82/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:56:02,988 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened f4e97eb825452964401fb63dd561ff82; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64529049, jitterRate=-0.03844223916530609}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T23:56:02,988 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:02,989 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for f4e97eb825452964401fb63dd561ff82: Running coprocessor pre-open hook at 1731369362968Writing region info on filesystem at 1731369362968Initializing all the Stores at 1731369362970 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369362970Cleaning up temporary data from old regions at 1731369362977 (+7 ms)Running coprocessor post-open hooks at 1731369362988 (+11 ms)Region opened successfully at 1731369362989 (+1 ms) 2024-11-11T23:56:02,994 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82., pid=27, masterSystemTime=1731369362962 2024-11-11T23:56:02,998 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. 2024-11-11T23:56:02,998 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. 2024-11-11T23:56:02,999 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=f4e97eb825452964401fb63dd561ff82, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:56:03,002 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4e97eb825452964401fb63dd561ff82, server=b70d29849faf,41471,1731369291585 because future has completed 2024-11-11T23:56:03,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-11T23:56:03,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure f4e97eb825452964401fb63dd561ff82, server=b70d29849faf,41471,1731369291585 in 194 msec 2024-11-11T23:56:03,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-11T23:56:03,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=f4e97eb825452964401fb63dd561ff82, ASSIGN in 354 msec 2024-11-11T23:56:03,011 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:56:03,011 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369363011"}]},"ts":"1731369363011"} 2024-11-11T23:56:03,014 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-11T23:56:03,016 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:56:03,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.1980 sec 2024-11-11T23:56:03,624 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:56:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T23:56:04,858 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-11T23:56:04,858 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-11T23:56:04,859 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:56:04,864 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-11T23:56:04,864 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:56:04,864 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-11T23:56:04,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82., hostname=b70d29849faf,41471,1731369291585, seqNum=2] 2024-11-11T23:56:04,878 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-11T23:56:04,878 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-11T23:56:04,881 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-11T23:56:04,883 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-11T23:56:04,958 INFO [master/b70d29849faf:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T23:56:04,958 INFO [master/b70d29849faf:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T23:56:06,297 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-11T23:56:10,887 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82., hostname=b70d29849faf,41471,1731369291585, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82., hostname=b70d29849faf,41471,1731369291585, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=b70d29849faf:41471 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6004ms, rpcTimeout=5999ms 2024-11-11T23:56:10,887 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82., hostname=b70d29849faf,41471,1731369291585, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6004ms, rpcTimeout=5999ms 2024-11-11T23:56:10,887 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82., hostname=b70d29849faf,41471,1731369291585, seqNum=2 from cache 2024-11-11T23:56:10,901 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=358 (was 332) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4419 (was 4424) 2024-11-11T23:56:10,909 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=358, ProcessCount=11, AvailableMemoryMB=4419 2024-11-11T23:56:10,912 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34765 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:56:10,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34765 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-11T23:56:10,914 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:56:10,915 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:10,915 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34765 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-11T23:56:10,916 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:56:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-11T23:56:10,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741851_1027 (size=407) 2024-11-11T23:56:10,927 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 802a7072f042b432b2f57f46902e38c1, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018 2024-11-11T23:56:10,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741852_1028 (size=58) 2024-11-11T23:56:10,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:56:10,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 802a7072f042b432b2f57f46902e38c1, disabling compactions & flushes 2024-11-11T23:56:10,942 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. 2024-11-11T23:56:10,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. 2024-11-11T23:56:10,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. after waiting 0 ms 2024-11-11T23:56:10,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. 2024-11-11T23:56:10,942 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. 2024-11-11T23:56:10,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 802a7072f042b432b2f57f46902e38c1: Waiting for close lock at 1731369370942Disabling compacts and flushes for region at 1731369370942Disabling writes for close at 1731369370942Writing region close event to WAL at 1731369370942Closed at 1731369370942 2024-11-11T23:56:10,944 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:56:10,944 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1731369370944"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369370944"}]},"ts":"1731369370944"} 2024-11-11T23:56:10,947 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:56:10,949 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:56:10,949 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369370949"}]},"ts":"1731369370949"} 2024-11-11T23:56:10,952 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-11T23:56:10,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=802a7072f042b432b2f57f46902e38c1, ASSIGN}] 2024-11-11T23:56:10,954 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=802a7072f042b432b2f57f46902e38c1, ASSIGN 2024-11-11T23:56:10,955 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=802a7072f042b432b2f57f46902e38c1, ASSIGN; state=OFFLINE, location=b70d29849faf,41471,1731369291585; forceNewPlan=false, retain=false 2024-11-11T23:56:11,106 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=802a7072f042b432b2f57f46902e38c1, regionState=OPENING, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:56:11,109 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=802a7072f042b432b2f57f46902e38c1, ASSIGN because future has completed 2024-11-11T23:56:11,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 802a7072f042b432b2f57f46902e38c1, server=b70d29849faf,41471,1731369291585}] 2024-11-11T23:56:11,269 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. 2024-11-11T23:56:11,270 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 802a7072f042b432b2f57f46902e38c1, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:56:11,270 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 802a7072f042b432b2f57f46902e38c1 2024-11-11T23:56:11,270 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:56:11,271 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 802a7072f042b432b2f57f46902e38c1 2024-11-11T23:56:11,271 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 802a7072f042b432b2f57f46902e38c1 2024-11-11T23:56:11,273 INFO [StoreOpener-802a7072f042b432b2f57f46902e38c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 802a7072f042b432b2f57f46902e38c1 2024-11-11T23:56:11,275 INFO [StoreOpener-802a7072f042b432b2f57f46902e38c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 802a7072f042b432b2f57f46902e38c1 columnFamilyName testFamily 2024-11-11T23:56:11,275 DEBUG [StoreOpener-802a7072f042b432b2f57f46902e38c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:11,275 INFO [StoreOpener-802a7072f042b432b2f57f46902e38c1-1 {}] regionserver.HStore(327): Store=802a7072f042b432b2f57f46902e38c1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:56:11,275 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 802a7072f042b432b2f57f46902e38c1 2024-11-11T23:56:11,276 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNext/802a7072f042b432b2f57f46902e38c1 2024-11-11T23:56:11,277 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNext/802a7072f042b432b2f57f46902e38c1 2024-11-11T23:56:11,277 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 802a7072f042b432b2f57f46902e38c1 2024-11-11T23:56:11,278 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 802a7072f042b432b2f57f46902e38c1 2024-11-11T23:56:11,280 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 802a7072f042b432b2f57f46902e38c1 2024-11-11T23:56:11,282 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNext/802a7072f042b432b2f57f46902e38c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:56:11,283 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 802a7072f042b432b2f57f46902e38c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65598212, jitterRate=-0.02251046895980835}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T23:56:11,283 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 802a7072f042b432b2f57f46902e38c1 2024-11-11T23:56:11,284 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 802a7072f042b432b2f57f46902e38c1: Running coprocessor pre-open hook at 1731369371271Writing region info on filesystem at 1731369371271Initializing all the Stores at 1731369371272 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369371272Cleaning up temporary data from old regions at 1731369371278 (+6 ms)Running coprocessor post-open hooks at 1731369371283 (+5 ms)Region opened successfully at 1731369371284 (+1 ms) 2024-11-11T23:56:11,285 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1., pid=30, masterSystemTime=1731369371262 2024-11-11T23:56:11,288 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. 2024-11-11T23:56:11,288 INFO [RS_OPEN_PRIORITY_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. 2024-11-11T23:56:11,288 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=802a7072f042b432b2f57f46902e38c1, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:56:11,291 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 802a7072f042b432b2f57f46902e38c1, server=b70d29849faf,41471,1731369291585 because future has completed 2024-11-11T23:56:11,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-11T23:56:11,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 802a7072f042b432b2f57f46902e38c1, server=b70d29849faf,41471,1731369291585 in 183 msec 2024-11-11T23:56:11,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-11T23:56:11,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=802a7072f042b432b2f57f46902e38c1, ASSIGN in 343 msec 2024-11-11T23:56:11,299 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:56:11,299 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369371299"}]},"ts":"1731369371299"} 2024-11-11T23:56:11,301 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-11T23:56:11,303 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:56:11,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 391 msec 2024-11-11T23:56:11,342 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-11T23:56:11,342 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-11T23:56:11,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-11T23:56:11,343 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-11T23:56:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-11T23:56:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-11T23:56:11,929 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-11T23:56:11,929 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-11T23:56:11,929 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:56:11,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-11T23:56:11,935 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:56:11,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-11T23:56:11,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1., hostname=b70d29849faf,41471,1731369291585, seqNum=2] 2024-11-11T23:56:11,950 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-11T23:56:11,950 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-11T23:56:11,954 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-11T23:56:11,956 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-11T23:56:11,957 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-11T23:56:11,958 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-11T23:56:12,299 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-11T23:56:16,802 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:56:17,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41471 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2070106616706891724' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-11T23:56:17,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41471 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:45872 deadline: 1731369383968, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2070106616706891724' does not exist 2024-11-11T23:56:17,974 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to b70d29849faf,41471,1731369291585 for closing scanner id = -2070106616706891724 for 802a7072f042b432b2f57f46902e38c1 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-2070106616706891724' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T23:56:17,982 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=377 (was 358) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4414 (was 4419) 2024-11-11T23:56:17,989 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=377, ProcessCount=11, AvailableMemoryMB=4414 2024-11-11T23:56:17,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T23:56:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-11T23:56:17,994 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T23:56:17,994 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:17,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-11T23:56:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T23:56:17,996 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T23:56:18,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741853_1029 (size=423) 2024-11-11T23:56:18,412 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 479afff78def2884291e8d7249d59876, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018 2024-11-11T23:56:18,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741854_1030 (size=74) 2024-11-11T23:56:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-11T23:56:18,824 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:56:18,824 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 479afff78def2884291e8d7249d59876, disabling compactions & flushes 2024-11-11T23:56:18,824 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. 2024-11-11T23:56:18,824 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. 2024-11-11T23:56:18,824 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. after waiting 0 ms 2024-11-11T23:56:18,824 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. 2024-11-11T23:56:18,824 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. 2024-11-11T23:56:18,824 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 479afff78def2884291e8d7249d59876: Waiting for close lock at 1731369378824Disabling compacts and flushes for region at 1731369378824Disabling writes for close at 1731369378824Writing region close event to WAL at 1731369378824Closed at 1731369378824 2024-11-11T23:56:18,827 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T23:56:18,827 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731369378827"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731369378827"}]},"ts":"1731369378827"} 2024-11-11T23:56:18,831 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T23:56:18,833 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T23:56:18,833 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369378833"}]},"ts":"1731369378833"} 2024-11-11T23:56:18,836 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-11T23:56:18,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=479afff78def2884291e8d7249d59876, ASSIGN}] 2024-11-11T23:56:18,839 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=479afff78def2884291e8d7249d59876, ASSIGN 2024-11-11T23:56:18,840 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=479afff78def2884291e8d7249d59876, ASSIGN; state=OFFLINE, location=b70d29849faf,41471,1731369291585; forceNewPlan=false, retain=false 2024-11-11T23:56:18,992 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=479afff78def2884291e8d7249d59876, regionState=OPENING, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:56:19,000 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=479afff78def2884291e8d7249d59876, ASSIGN because future has completed 2024-11-11T23:56:19,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 479afff78def2884291e8d7249d59876, server=b70d29849faf,41471,1731369291585}] 2024-11-11T23:56:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T23:56:19,168 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. 2024-11-11T23:56:19,169 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 479afff78def2884291e8d7249d59876, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876.', STARTKEY => '', ENDKEY => ''} 2024-11-11T23:56:19,169 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 479afff78def2884291e8d7249d59876 2024-11-11T23:56:19,169 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T23:56:19,169 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 479afff78def2884291e8d7249d59876 2024-11-11T23:56:19,170 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 479afff78def2884291e8d7249d59876 2024-11-11T23:56:19,172 INFO [StoreOpener-479afff78def2884291e8d7249d59876-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 479afff78def2884291e8d7249d59876 2024-11-11T23:56:19,174 INFO [StoreOpener-479afff78def2884291e8d7249d59876-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 479afff78def2884291e8d7249d59876 columnFamilyName testFamily 2024-11-11T23:56:19,174 DEBUG [StoreOpener-479afff78def2884291e8d7249d59876-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T23:56:19,175 INFO [StoreOpener-479afff78def2884291e8d7249d59876-1 {}] regionserver.HStore(327): Store=479afff78def2884291e8d7249d59876/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T23:56:19,175 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 479afff78def2884291e8d7249d59876 2024-11-11T23:56:19,176 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScannerAsync/479afff78def2884291e8d7249d59876 2024-11-11T23:56:19,177 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScannerAsync/479afff78def2884291e8d7249d59876 2024-11-11T23:56:19,178 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 479afff78def2884291e8d7249d59876 2024-11-11T23:56:19,178 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 479afff78def2884291e8d7249d59876 2024-11-11T23:56:19,181 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 479afff78def2884291e8d7249d59876 2024-11-11T23:56:19,184 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScannerAsync/479afff78def2884291e8d7249d59876/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T23:56:19,185 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 479afff78def2884291e8d7249d59876; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63351279, jitterRate=-0.055992379784584045}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T23:56:19,185 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 479afff78def2884291e8d7249d59876 2024-11-11T23:56:19,186 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 479afff78def2884291e8d7249d59876: Running coprocessor pre-open hook at 1731369379170Writing region info on filesystem at 1731369379170Initializing all the Stores at 1731369379171 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731369379171Cleaning up temporary data from old regions at 1731369379178 (+7 ms)Running coprocessor post-open hooks at 1731369379185 (+7 ms)Region opened successfully at 1731369379186 (+1 ms) 2024-11-11T23:56:19,187 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., pid=33, masterSystemTime=1731369379155 2024-11-11T23:56:19,189 DEBUG [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. 2024-11-11T23:56:19,190 INFO [RS_OPEN_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. 2024-11-11T23:56:19,190 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=479afff78def2884291e8d7249d59876, regionState=OPEN, openSeqNum=2, regionLocation=b70d29849faf,41471,1731369291585 2024-11-11T23:56:19,193 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 479afff78def2884291e8d7249d59876, server=b70d29849faf,41471,1731369291585 because future has completed 2024-11-11T23:56:19,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-11T23:56:19,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 479afff78def2884291e8d7249d59876, server=b70d29849faf,41471,1731369291585 in 193 msec 2024-11-11T23:56:19,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-11T23:56:19,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=479afff78def2884291e8d7249d59876, ASSIGN in 360 msec 2024-11-11T23:56:19,200 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T23:56:19,200 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731369379200"}]},"ts":"1731369379200"} 2024-11-11T23:56:19,202 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-11T23:56:19,203 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T23:56:19,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2130 sec 2024-11-11T23:56:20,165 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T23:56:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34765 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T23:56:21,022 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-11T23:56:21,022 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-11T23:56:21,023 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:56:21,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-11T23:56:21,034 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T23:56:21,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-11T23:56:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., hostname=b70d29849faf,41471,1731369291585, seqNum=2] 2024-11-11T23:56:21,049 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-11T23:56:21,049 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-11T23:56:21,052 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-11T23:56:21,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-11T23:56:21,342 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-11T23:56:21,342 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-11T23:56:22,058 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., hostname=b70d29849faf,41471,1731369291585, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., hostname=b70d29849faf,41471,1731369291585, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=b70d29849faf:41471 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-11-11T23:56:22,058 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., hostname=b70d29849faf,41471,1731369291585, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-11-11T23:56:22,058 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., hostname=b70d29849faf,41471,1731369291585, seqNum=2 from cache 2024-11-11T23:56:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-11T23:56:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., hostname=b70d29849faf,41471,1731369291585, seqNum=2] 2024-11-11T23:56:23,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41471 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-11T23:56:24,058 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., hostname=b70d29849faf,41471,1731369291585, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., hostname=b70d29849faf,41471,1731369291585, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=b70d29849faf:41471 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=977ms, rpcTimeout=972ms 2024-11-11T23:56:24,058 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., hostname=b70d29849faf,41471,1731369291585, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=977ms, rpcTimeout=972ms 2024-11-11T23:56:24,058 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., hostname=b70d29849faf,41471,1731369291585, seqNum=2 from cache 2024-11-11T23:56:24,075 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=334 (was 377), ProcessCount=11 (was 11), AvailableMemoryMB=4410 (was 4414) 2024-11-11T23:56:24,075 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T23:56:24,076 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T23:56:24,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:24,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:24,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:56:24,081 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T23:56:24,081 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T23:56:24,082 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T23:56:24,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:24,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:24,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T23:56:24,082 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T23:56:24,082 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=688211475, stopped=false 2024-11-11T23:56:24,082 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b70d29849faf,34765,1731369291015 2024-11-11T23:56:24,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T23:56:24,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T23:56:24,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:56:24,084 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T23:56:24,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:56:24,084 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T23:56:24,085 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T23:56:24,085 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T23:56:24,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:24,085 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T23:56:24,085 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b70d29849faf,41471,1731369291585' ***** 2024-11-11T23:56:24,085 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T23:56:24,086 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T23:56:24,086 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T23:56:24,086 INFO [RS:0;b70d29849faf:41471 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T23:56:24,086 INFO [RS:0;b70d29849faf:41471 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T23:56:24,086 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(3091): Received CLOSE for 65ce0dbfc1952458af49fd41d21f0a5d 2024-11-11T23:56:24,087 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(3091): Received CLOSE for 258cf9e310212682e64135fb4ca21e76 2024-11-11T23:56:24,087 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(3091): Received CLOSE for 4ea1681b2f6bdcab4ccb742ef1cfbcde 2024-11-11T23:56:24,087 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(3091): Received CLOSE for f42f7a1136d0e434e5b2c914b87b4653 2024-11-11T23:56:24,087 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(3091): Received CLOSE for 17720cb52cfe4bc507a89874ac61d582 2024-11-11T23:56:24,087 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(3091): Received CLOSE for f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:24,087 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(3091): Received CLOSE for 6c54b823914e3bebb023069112e27160 2024-11-11T23:56:24,087 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(3091): Received CLOSE for 479afff78def2884291e8d7249d59876 2024-11-11T23:56:24,087 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(3091): Received CLOSE for 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:56:24,087 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 65ce0dbfc1952458af49fd41d21f0a5d, disabling compactions & flushes 2024-11-11T23:56:24,087 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(3091): Received CLOSE for 802a7072f042b432b2f57f46902e38c1 2024-11-11T23:56:24,087 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. 2024-11-11T23:56:24,087 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. 2024-11-11T23:56:24,087 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(959): stopping server b70d29849faf,41471,1731369291585 2024-11-11T23:56:24,087 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. after waiting 0 ms 2024-11-11T23:56:24,087 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. 2024-11-11T23:56:24,087 INFO [RS:0;b70d29849faf:41471 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T23:56:24,088 INFO [RS:0;b70d29849faf:41471 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b70d29849faf:41471. 2024-11-11T23:56:24,088 DEBUG [RS:0;b70d29849faf:41471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T23:56:24,088 DEBUG [RS:0;b70d29849faf:41471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:24,088 INFO [RS:0;b70d29849faf:41471 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T23:56:24,088 INFO [RS:0;b70d29849faf:41471 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T23:56:24,088 INFO [RS:0;b70d29849faf:41471 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T23:56:24,088 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T23:56:24,089 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-11T23:56:24,089 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T23:56:24,089 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T23:56:24,089 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T23:56:24,089 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T23:56:24,089 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T23:56:24,089 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17720cb52cfe4bc507a89874ac61d582, 258cf9e310212682e64135fb4ca21e76, 479afff78def2884291e8d7249d59876, 4ea1681b2f6bdcab4ccb742ef1cfbcde, 65ce0dbfc1952458af49fd41d21f0a5d, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83, f42f7a1136d0e434e5b2c914b87b4653, f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:24,091 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 65ce0dbfc1952458af49fd41d21f0a5d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-11T23:56:24,091 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-11T23:56:24,142 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScanner/65ce0dbfc1952458af49fd41d21f0a5d/.tmp/testFamily/117c230a7bd24568aaf8a4c70913dd37 is 61, key is row-0/testFamily:testQualifier/1731369348588/Put/seqid=0 2024-11-11T23:56:24,148 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/.tmp/info/8c1435361c6b4e2fa0016df3cd3a8ae3 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde./info:regioninfo/1731369325879/Put/seqid=0 2024-11-11T23:56:24,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741855_1031 (size=5049) 2024-11-11T23:56:24,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741856_1032 (size=20564) 2024-11-11T23:56:24,289 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17720cb52cfe4bc507a89874ac61d582, 258cf9e310212682e64135fb4ca21e76, 479afff78def2884291e8d7249d59876, 4ea1681b2f6bdcab4ccb742ef1cfbcde, 65ce0dbfc1952458af49fd41d21f0a5d, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83, f42f7a1136d0e434e5b2c914b87b4653, f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:24,333 INFO [regionserver/b70d29849faf:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T23:56:24,333 INFO [regionserver/b70d29849faf:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T23:56:24,490 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17720cb52cfe4bc507a89874ac61d582, 258cf9e310212682e64135fb4ca21e76, 479afff78def2884291e8d7249d59876, 4ea1681b2f6bdcab4ccb742ef1cfbcde, 65ce0dbfc1952458af49fd41d21f0a5d, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83, f42f7a1136d0e434e5b2c914b87b4653, f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:24,555 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScanner/65ce0dbfc1952458af49fd41d21f0a5d/.tmp/testFamily/117c230a7bd24568aaf8a4c70913dd37 2024-11-11T23:56:24,557 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/.tmp/info/8c1435361c6b4e2fa0016df3cd3a8ae3 2024-11-11T23:56:24,581 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-11T23:56:24,603 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScanner/65ce0dbfc1952458af49fd41d21f0a5d/.tmp/testFamily/117c230a7bd24568aaf8a4c70913dd37 as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScanner/65ce0dbfc1952458af49fd41d21f0a5d/testFamily/117c230a7bd24568aaf8a4c70913dd37 2024-11-11T23:56:24,612 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScanner/65ce0dbfc1952458af49fd41d21f0a5d/testFamily/117c230a7bd24568aaf8a4c70913dd37, entries=4, sequenceid=8, filesize=4.9 K 2024-11-11T23:56:24,620 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/.tmp/ns/b3520a51245f4ba6b38eabfcb3ef25a4 is 43, key is default/ns:d/1731369324815/Put/seqid=0 2024-11-11T23:56:24,621 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 65ce0dbfc1952458af49fd41d21f0a5d in 529ms, sequenceid=8, compaction requested=false 2024-11-11T23:56:24,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741857_1033 (size=5153) 2024-11-11T23:56:24,628 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScanner/65ce0dbfc1952458af49fd41d21f0a5d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-11T23:56:24,630 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. 2024-11-11T23:56:24,630 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 65ce0dbfc1952458af49fd41d21f0a5d: Waiting for close lock at 1731369384087Running coprocessor pre-close hooks at 1731369384087Disabling compacts and flushes for region at 1731369384087Disabling writes for close at 1731369384087Obtaining lock to block concurrent updates at 1731369384091 (+4 ms)Preparing flush snapshotting stores in 65ce0dbfc1952458af49fd41d21f0a5d at 1731369384091Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731369384098 (+7 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. at 1731369384100 (+2 ms)Flushing 65ce0dbfc1952458af49fd41d21f0a5d/testFamily: creating writer at 1731369384101 (+1 ms)Flushing 65ce0dbfc1952458af49fd41d21f0a5d/testFamily: appending metadata at 1731369384137 (+36 ms)Flushing 65ce0dbfc1952458af49fd41d21f0a5d/testFamily: closing flushed file at 1731369384139 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b170964: reopening flushed file at 1731369384602 (+463 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 65ce0dbfc1952458af49fd41d21f0a5d in 529ms, sequenceid=8, compaction requested=false at 1731369384621 (+19 ms)Writing region close event to WAL at 1731369384623 (+2 ms)Running coprocessor post-close hooks at 1731369384628 (+5 ms)Closed at 1731369384630 (+2 ms) 2024-11-11T23:56:24,630 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1731369345525.65ce0dbfc1952458af49fd41d21f0a5d. 2024-11-11T23:56:24,631 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 258cf9e310212682e64135fb4ca21e76, disabling compactions & flushes 2024-11-11T23:56:24,631 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. 2024-11-11T23:56:24,631 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. 2024-11-11T23:56:24,631 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. after waiting 0 ms 2024-11-11T23:56:24,631 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. 2024-11-11T23:56:24,631 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 258cf9e310212682e64135fb4ca21e76 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-11T23:56:24,648 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNextAsync/258cf9e310212682e64135fb4ca21e76/.tmp/testFamily/ecf872cb0b2b4f57aff88229d44edef1 is 61, key is row-0/testFamily:testQualifier/1731369354690/Put/seqid=0 2024-11-11T23:56:24,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741858_1034 (size=5049) 2024-11-11T23:56:24,671 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:56:24,690 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17720cb52cfe4bc507a89874ac61d582, 258cf9e310212682e64135fb4ca21e76, 479afff78def2884291e8d7249d59876, 4ea1681b2f6bdcab4ccb742ef1cfbcde, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83, f42f7a1136d0e434e5b2c914b87b4653, f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:24,720 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-11T23:56:24,869 DEBUG [master/b70d29849faf:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-11T23:56:24,891 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17720cb52cfe4bc507a89874ac61d582, 258cf9e310212682e64135fb4ca21e76, 479afff78def2884291e8d7249d59876, 4ea1681b2f6bdcab4ccb742ef1cfbcde, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83, f42f7a1136d0e434e5b2c914b87b4653, f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:25,026 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/.tmp/ns/b3520a51245f4ba6b38eabfcb3ef25a4 2024-11-11T23:56:25,053 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/.tmp/table/dd580634013f44f8ad5960700bf0e340 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1731369325903/Put/seqid=0 2024-11-11T23:56:25,054 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNextAsync/258cf9e310212682e64135fb4ca21e76/.tmp/testFamily/ecf872cb0b2b4f57aff88229d44edef1 2024-11-11T23:56:25,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741859_1035 (size=6699) 2024-11-11T23:56:25,063 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNextAsync/258cf9e310212682e64135fb4ca21e76/.tmp/testFamily/ecf872cb0b2b4f57aff88229d44edef1 as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNextAsync/258cf9e310212682e64135fb4ca21e76/testFamily/ecf872cb0b2b4f57aff88229d44edef1 2024-11-11T23:56:25,071 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNextAsync/258cf9e310212682e64135fb4ca21e76/testFamily/ecf872cb0b2b4f57aff88229d44edef1, entries=4, sequenceid=8, filesize=4.9 K 2024-11-11T23:56:25,072 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 258cf9e310212682e64135fb4ca21e76 in 441ms, sequenceid=8, compaction requested=false 2024-11-11T23:56:25,076 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNextAsync/258cf9e310212682e64135fb4ca21e76/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-11T23:56:25,077 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. 2024-11-11T23:56:25,077 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 258cf9e310212682e64135fb4ca21e76: Waiting for close lock at 1731369384630Running coprocessor pre-close hooks at 1731369384630Disabling compacts and flushes for region at 1731369384630Disabling writes for close at 1731369384631 (+1 ms)Obtaining lock to block concurrent updates at 1731369384631Preparing flush snapshotting stores in 258cf9e310212682e64135fb4ca21e76 at 1731369384631Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731369384631Flushing stores of testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. at 1731369384632 (+1 ms)Flushing 258cf9e310212682e64135fb4ca21e76/testFamily: creating writer at 1731369384632Flushing 258cf9e310212682e64135fb4ca21e76/testFamily: appending metadata at 1731369384647 (+15 ms)Flushing 258cf9e310212682e64135fb4ca21e76/testFamily: closing flushed file at 1731369384647Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@205568c5: reopening flushed file at 1731369385062 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 258cf9e310212682e64135fb4ca21e76 in 441ms, sequenceid=8, compaction requested=false at 1731369385072 (+10 ms)Writing region close event to WAL at 1731369385072Running coprocessor post-close hooks at 1731369385077 (+5 ms)Closed at 1731369385077 2024-11-11T23:56:25,077 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1731369351632.258cf9e310212682e64135fb4ca21e76. 2024-11-11T23:56:25,077 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4ea1681b2f6bdcab4ccb742ef1cfbcde, disabling compactions & flushes 2024-11-11T23:56:25,077 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. 2024-11-11T23:56:25,077 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. 2024-11-11T23:56:25,077 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. after waiting 0 ms 2024-11-11T23:56:25,077 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. 2024-11-11T23:56:25,078 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4ea1681b2f6bdcab4ccb742ef1cfbcde 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-11T23:56:25,091 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-11T23:56:25,091 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1325): Online Regions={4ea1681b2f6bdcab4ccb742ef1cfbcde=testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde., f42f7a1136d0e434e5b2c914b87b4653=hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653., 17720cb52cfe4bc507a89874ac61d582=testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582., f4e97eb825452964401fb63dd561ff82=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82., 6c54b823914e3bebb023069112e27160=testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160., 1588230740=hbase:meta,,1.1588230740, 479afff78def2884291e8d7249d59876=testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., 981c9fbb3e7f4c993ecad2d02d339d83=hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83., 802a7072f042b432b2f57f46902e38c1=hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1.} 2024-11-11T23:56:25,091 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17720cb52cfe4bc507a89874ac61d582, 479afff78def2884291e8d7249d59876, 4ea1681b2f6bdcab4ccb742ef1cfbcde, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83, f42f7a1136d0e434e5b2c914b87b4653, f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:25,093 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4ea1681b2f6bdcab4ccb742ef1cfbcde/.tmp/testFamily/6f2e500029704aa7b2d45a80fe13fe6c is 61, key is row-0/testFamily:testQualifier/1731369326094/Put/seqid=0 2024-11-11T23:56:25,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741860_1036 (size=5049) 2024-11-11T23:56:25,291 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17720cb52cfe4bc507a89874ac61d582, 479afff78def2884291e8d7249d59876, 4ea1681b2f6bdcab4ccb742ef1cfbcde, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83, f42f7a1136d0e434e5b2c914b87b4653, f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:25,462 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/.tmp/table/dd580634013f44f8ad5960700bf0e340 2024-11-11T23:56:25,477 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/.tmp/info/8c1435361c6b4e2fa0016df3cd3a8ae3 as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/info/8c1435361c6b4e2fa0016df3cd3a8ae3 2024-11-11T23:56:25,486 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/info/8c1435361c6b4e2fa0016df3cd3a8ae3, entries=100, sequenceid=56, filesize=20.1 K 2024-11-11T23:56:25,488 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/.tmp/ns/b3520a51245f4ba6b38eabfcb3ef25a4 as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/ns/b3520a51245f4ba6b38eabfcb3ef25a4 2024-11-11T23:56:25,492 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17720cb52cfe4bc507a89874ac61d582, 479afff78def2884291e8d7249d59876, 4ea1681b2f6bdcab4ccb742ef1cfbcde, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83, f42f7a1136d0e434e5b2c914b87b4653, f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:25,497 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/ns/b3520a51245f4ba6b38eabfcb3ef25a4, entries=2, sequenceid=56, filesize=5.0 K 2024-11-11T23:56:25,498 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4ea1681b2f6bdcab4ccb742ef1cfbcde/.tmp/testFamily/6f2e500029704aa7b2d45a80fe13fe6c 2024-11-11T23:56:25,499 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/.tmp/table/dd580634013f44f8ad5960700bf0e340 as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/table/dd580634013f44f8ad5960700bf0e340 2024-11-11T23:56:25,506 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/table/dd580634013f44f8ad5960700bf0e340, entries=20, sequenceid=56, filesize=6.5 K 2024-11-11T23:56:25,507 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1418ms, sequenceid=56, compaction requested=false 2024-11-11T23:56:25,507 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4ea1681b2f6bdcab4ccb742ef1cfbcde/.tmp/testFamily/6f2e500029704aa7b2d45a80fe13fe6c as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4ea1681b2f6bdcab4ccb742ef1cfbcde/testFamily/6f2e500029704aa7b2d45a80fe13fe6c 2024-11-11T23:56:25,513 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-11T23:56:25,514 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T23:56:25,515 INFO [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T23:56:25,515 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731369384089Running coprocessor pre-close hooks at 1731369384089Disabling compacts and flushes for region at 1731369384089Disabling writes for close at 1731369384089Obtaining lock to block concurrent updates at 1731369384091 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1731369384091Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1731369384098 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731369384100 (+2 ms)Flushing 1588230740/info: creating writer at 1731369384101 (+1 ms)Flushing 1588230740/info: appending metadata at 1731369384143 (+42 ms)Flushing 1588230740/info: closing flushed file at 1731369384143Flushing 1588230740/ns: creating writer at 1731369384602 (+459 ms)Flushing 1588230740/ns: appending metadata at 1731369384620 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731369384620Flushing 1588230740/table: creating writer at 1731369385036 (+416 ms)Flushing 1588230740/table: appending metadata at 1731369385052 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731369385053 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cb3b19: reopening flushed file at 1731369385476 (+423 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@355f4893: reopening flushed file at 1731369385487 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f7958eb: reopening flushed file at 1731369385497 (+10 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1418ms, sequenceid=56, compaction requested=false at 1731369385507 (+10 ms)Writing region close event to WAL at 1731369385509 (+2 ms)Running coprocessor post-close hooks at 1731369385514 (+5 ms)Closed at 1731369385514 2024-11-11T23:56:25,515 DEBUG [RS_CLOSE_META-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T23:56:25,516 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4ea1681b2f6bdcab4ccb742ef1cfbcde/testFamily/6f2e500029704aa7b2d45a80fe13fe6c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-11T23:56:25,517 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4ea1681b2f6bdcab4ccb742ef1cfbcde in 439ms, sequenceid=8, compaction requested=false 2024-11-11T23:56:25,522 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4ea1681b2f6bdcab4ccb742ef1cfbcde/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-11T23:56:25,523 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. 2024-11-11T23:56:25,523 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4ea1681b2f6bdcab4ccb742ef1cfbcde: Waiting for close lock at 1731369385077Running coprocessor pre-close hooks at 1731369385077Disabling compacts and flushes for region at 1731369385077Disabling writes for close at 1731369385077Obtaining lock to block concurrent updates at 1731369385078 (+1 ms)Preparing flush snapshotting stores in 4ea1681b2f6bdcab4ccb742ef1cfbcde at 1731369385078Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731369385078Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. at 1731369385078Flushing 4ea1681b2f6bdcab4ccb742ef1cfbcde/testFamily: creating writer at 1731369385079 (+1 ms)Flushing 4ea1681b2f6bdcab4ccb742ef1cfbcde/testFamily: appending metadata at 1731369385092 (+13 ms)Flushing 4ea1681b2f6bdcab4ccb742ef1cfbcde/testFamily: closing flushed file at 1731369385093 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f6f3fa: reopening flushed file at 1731369385506 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4ea1681b2f6bdcab4ccb742ef1cfbcde in 439ms, sequenceid=8, compaction requested=false at 1731369385517 (+11 ms)Writing region close event to WAL at 1731369385518 (+1 ms)Running coprocessor post-close hooks at 1731369385523 (+5 ms)Closed at 1731369385523 2024-11-11T23:56:25,523 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731369325014.4ea1681b2f6bdcab4ccb742ef1cfbcde. 2024-11-11T23:56:25,523 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f42f7a1136d0e434e5b2c914b87b4653, disabling compactions & flushes 2024-11-11T23:56:25,523 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. 2024-11-11T23:56:25,524 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. 2024-11-11T23:56:25,524 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. after waiting 0 ms 2024-11-11T23:56:25,524 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. 2024-11-11T23:56:25,524 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f42f7a1136d0e434e5b2c914b87b4653 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-11T23:56:25,541 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScanner/f42f7a1136d0e434e5b2c914b87b4653/.tmp/testFamily/4fb65ddb5a364efeaaa90f4f47ecf1c8 is 61, key is row-0/testFamily:testQualifier/1731369329290/Put/seqid=0 2024-11-11T23:56:25,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741861_1037 (size=5049) 2024-11-11T23:56:25,692 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 17720cb52cfe4bc507a89874ac61d582, 479afff78def2884291e8d7249d59876, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83, f42f7a1136d0e434e5b2c914b87b4653, f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:25,893 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 17720cb52cfe4bc507a89874ac61d582, 479afff78def2884291e8d7249d59876, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83, f42f7a1136d0e434e5b2c914b87b4653, f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:25,948 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScanner/f42f7a1136d0e434e5b2c914b87b4653/.tmp/testFamily/4fb65ddb5a364efeaaa90f4f47ecf1c8 2024-11-11T23:56:25,960 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScanner/f42f7a1136d0e434e5b2c914b87b4653/.tmp/testFamily/4fb65ddb5a364efeaaa90f4f47ecf1c8 as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScanner/f42f7a1136d0e434e5b2c914b87b4653/testFamily/4fb65ddb5a364efeaaa90f4f47ecf1c8 2024-11-11T23:56:25,969 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScanner/f42f7a1136d0e434e5b2c914b87b4653/testFamily/4fb65ddb5a364efeaaa90f4f47ecf1c8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-11T23:56:25,971 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f42f7a1136d0e434e5b2c914b87b4653 in 446ms, sequenceid=8, compaction requested=false 2024-11-11T23:56:25,975 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScanner/f42f7a1136d0e434e5b2c914b87b4653/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-11T23:56:25,976 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. 2024-11-11T23:56:25,976 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f42f7a1136d0e434e5b2c914b87b4653: Waiting for close lock at 1731369385523Running coprocessor pre-close hooks at 1731369385523Disabling compacts and flushes for region at 1731369385523Disabling writes for close at 1731369385524 (+1 ms)Obtaining lock to block concurrent updates at 1731369385524Preparing flush snapshotting stores in f42f7a1136d0e434e5b2c914b87b4653 at 1731369385524Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731369385524Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. at 1731369385525 (+1 ms)Flushing f42f7a1136d0e434e5b2c914b87b4653/testFamily: creating writer at 1731369385525Flushing f42f7a1136d0e434e5b2c914b87b4653/testFamily: appending metadata at 1731369385541 (+16 ms)Flushing f42f7a1136d0e434e5b2c914b87b4653/testFamily: closing flushed file at 1731369385541Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33118607: reopening flushed file at 1731369385959 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f42f7a1136d0e434e5b2c914b87b4653 in 446ms, sequenceid=8, compaction requested=false at 1731369385971 (+12 ms)Writing region close event to WAL at 1731369385971Running coprocessor post-close hooks at 1731369385976 (+5 ms)Closed at 1731369385976 2024-11-11T23:56:25,977 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731369326249.f42f7a1136d0e434e5b2c914b87b4653. 2024-11-11T23:56:25,977 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 17720cb52cfe4bc507a89874ac61d582, disabling compactions & flushes 2024-11-11T23:56:25,977 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. 2024-11-11T23:56:25,977 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. 2024-11-11T23:56:25,977 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. after waiting 0 ms 2024-11-11T23:56:25,977 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. 2024-11-11T23:56:25,977 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 17720cb52cfe4bc507a89874ac61d582 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-11T23:56:25,995 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextException/17720cb52cfe4bc507a89874ac61d582/.tmp/testFamily/73c3911eb0c64d7599dacc94cea9fe22 is 61, key is row-0/testFamily:testQualifier/1731369345468/Put/seqid=0 2024-11-11T23:56:25,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741862_1038 (size=5049) 2024-11-11T23:56:26,093 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-11T23:56:26,093 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1325): Online Regions={17720cb52cfe4bc507a89874ac61d582=testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582., f4e97eb825452964401fb63dd561ff82=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82., 6c54b823914e3bebb023069112e27160=testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160., 479afff78def2884291e8d7249d59876=testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., 981c9fbb3e7f4c993ecad2d02d339d83=hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83., 802a7072f042b432b2f57f46902e38c1=hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1.} 2024-11-11T23:56:26,093 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 17720cb52cfe4bc507a89874ac61d582, 479afff78def2884291e8d7249d59876, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83, f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:26,294 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 17720cb52cfe4bc507a89874ac61d582, 479afff78def2884291e8d7249d59876, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83, f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:26,401 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextException/17720cb52cfe4bc507a89874ac61d582/.tmp/testFamily/73c3911eb0c64d7599dacc94cea9fe22 2024-11-11T23:56:26,412 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextException/17720cb52cfe4bc507a89874ac61d582/.tmp/testFamily/73c3911eb0c64d7599dacc94cea9fe22 as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextException/17720cb52cfe4bc507a89874ac61d582/testFamily/73c3911eb0c64d7599dacc94cea9fe22 2024-11-11T23:56:26,422 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextException/17720cb52cfe4bc507a89874ac61d582/testFamily/73c3911eb0c64d7599dacc94cea9fe22, entries=4, sequenceid=8, filesize=4.9 K 2024-11-11T23:56:26,423 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 17720cb52cfe4bc507a89874ac61d582 in 446ms, sequenceid=8, compaction requested=false 2024-11-11T23:56:26,428 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testRetryOutOfOrderScannerNextException/17720cb52cfe4bc507a89874ac61d582/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-11T23:56:26,429 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. 2024-11-11T23:56:26,429 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 17720cb52cfe4bc507a89874ac61d582: Waiting for close lock at 1731369385977Running coprocessor pre-close hooks at 1731369385977Disabling compacts and flushes for region at 1731369385977Disabling writes for close at 1731369385977Obtaining lock to block concurrent updates at 1731369385977Preparing flush snapshotting stores in 17720cb52cfe4bc507a89874ac61d582 at 1731369385977Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731369385978 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. at 1731369385978Flushing 17720cb52cfe4bc507a89874ac61d582/testFamily: creating writer at 1731369385979 (+1 ms)Flushing 17720cb52cfe4bc507a89874ac61d582/testFamily: appending metadata at 1731369385994 (+15 ms)Flushing 17720cb52cfe4bc507a89874ac61d582/testFamily: closing flushed file at 1731369385994Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@526ccd34: reopening flushed file at 1731369386411 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 17720cb52cfe4bc507a89874ac61d582 in 446ms, sequenceid=8, compaction requested=false at 1731369386423 (+12 ms)Writing region close event to WAL at 1731369386424 (+1 ms)Running coprocessor post-close hooks at 1731369386429 (+5 ms)Closed at 1731369386429 2024-11-11T23:56:26,429 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1731369342429.17720cb52cfe4bc507a89874ac61d582. 2024-11-11T23:56:26,429 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f4e97eb825452964401fb63dd561ff82, disabling compactions & flushes 2024-11-11T23:56:26,429 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. 2024-11-11T23:56:26,429 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. 2024-11-11T23:56:26,429 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. after waiting 0 ms 2024-11-11T23:56:26,430 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. 2024-11-11T23:56:26,430 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f4e97eb825452964401fb63dd561ff82 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-11T23:56:26,451 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f4e97eb825452964401fb63dd561ff82/.tmp/testFamily/2b01f9c37b694c35afc5823a84e3a0c5 is 61, key is row-0/testFamily:testQualifier/1731369364868/Put/seqid=0 2024-11-11T23:56:26,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741863_1039 (size=5049) 2024-11-11T23:56:26,494 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 479afff78def2884291e8d7249d59876, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83, f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:26,695 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 479afff78def2884291e8d7249d59876, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83, f4e97eb825452964401fb63dd561ff82 2024-11-11T23:56:26,857 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f4e97eb825452964401fb63dd561ff82/.tmp/testFamily/2b01f9c37b694c35afc5823a84e3a0c5 2024-11-11T23:56:26,866 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f4e97eb825452964401fb63dd561ff82/.tmp/testFamily/2b01f9c37b694c35afc5823a84e3a0c5 as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f4e97eb825452964401fb63dd561ff82/testFamily/2b01f9c37b694c35afc5823a84e3a0c5 2024-11-11T23:56:26,874 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f4e97eb825452964401fb63dd561ff82/testFamily/2b01f9c37b694c35afc5823a84e3a0c5, entries=4, sequenceid=8, filesize=4.9 K 2024-11-11T23:56:26,875 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f4e97eb825452964401fb63dd561ff82 in 445ms, sequenceid=8, compaction requested=false 2024-11-11T23:56:26,880 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f4e97eb825452964401fb63dd561ff82/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-11T23:56:26,881 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. 2024-11-11T23:56:26,881 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f4e97eb825452964401fb63dd561ff82: Waiting for close lock at 1731369386429Running coprocessor pre-close hooks at 1731369386429Disabling compacts and flushes for region at 1731369386429Disabling writes for close at 1731369386429Obtaining lock to block concurrent updates at 1731369386430 (+1 ms)Preparing flush snapshotting stores in f4e97eb825452964401fb63dd561ff82 at 1731369386430Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731369386430Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. at 1731369386431 (+1 ms)Flushing f4e97eb825452964401fb63dd561ff82/testFamily: creating writer at 1731369386431Flushing f4e97eb825452964401fb63dd561ff82/testFamily: appending metadata at 1731369386450 (+19 ms)Flushing f4e97eb825452964401fb63dd561ff82/testFamily: closing flushed file at 1731369386450Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34195e04: reopening flushed file at 1731369386865 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f4e97eb825452964401fb63dd561ff82 in 445ms, sequenceid=8, compaction requested=false at 1731369386875 (+10 ms)Writing region close event to WAL at 1731369386876 (+1 ms)Running coprocessor post-close hooks at 1731369386880 (+4 ms)Closed at 1731369386880 2024-11-11T23:56:26,881 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731369361817.f4e97eb825452964401fb63dd561ff82. 2024-11-11T23:56:26,881 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6c54b823914e3bebb023069112e27160, disabling compactions & flushes 2024-11-11T23:56:26,881 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. 2024-11-11T23:56:26,881 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. 2024-11-11T23:56:26,881 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. after waiting 0 ms 2024-11-11T23:56:26,881 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. 2024-11-11T23:56:26,881 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6c54b823914e3bebb023069112e27160 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-11T23:56:26,895 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 479afff78def2884291e8d7249d59876, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:56:26,898 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNext/6c54b823914e3bebb023069112e27160/.tmp/testFamily/fffe1a6d685b414ca1688cc110b8849d is 61, key is row-0/testFamily:testQualifier/1731369358770/Put/seqid=0 2024-11-11T23:56:26,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741864_1040 (size=5049) 2024-11-11T23:56:27,095 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-11T23:56:27,095 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1325): Online Regions={6c54b823914e3bebb023069112e27160=testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160., 479afff78def2884291e8d7249d59876=testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., 981c9fbb3e7f4c993ecad2d02d339d83=hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83., 802a7072f042b432b2f57f46902e38c1=hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1.} 2024-11-11T23:56:27,095 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 479afff78def2884291e8d7249d59876, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:56:27,296 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 479afff78def2884291e8d7249d59876, 6c54b823914e3bebb023069112e27160, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:56:27,305 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNext/6c54b823914e3bebb023069112e27160/.tmp/testFamily/fffe1a6d685b414ca1688cc110b8849d 2024-11-11T23:56:27,319 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNext/6c54b823914e3bebb023069112e27160/.tmp/testFamily/fffe1a6d685b414ca1688cc110b8849d as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNext/6c54b823914e3bebb023069112e27160/testFamily/fffe1a6d685b414ca1688cc110b8849d 2024-11-11T23:56:27,327 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNext/6c54b823914e3bebb023069112e27160/testFamily/fffe1a6d685b414ca1688cc110b8849d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-11T23:56:27,329 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6c54b823914e3bebb023069112e27160 in 447ms, sequenceid=8, compaction requested=false 2024-11-11T23:56:27,333 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnNext/6c54b823914e3bebb023069112e27160/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-11T23:56:27,334 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. 2024-11-11T23:56:27,334 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6c54b823914e3bebb023069112e27160: Waiting for close lock at 1731369386881Running coprocessor pre-close hooks at 1731369386881Disabling compacts and flushes for region at 1731369386881Disabling writes for close at 1731369386881Obtaining lock to block concurrent updates at 1731369386881Preparing flush snapshotting stores in 6c54b823914e3bebb023069112e27160 at 1731369386881Finished memstore snapshotting testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731369386882 (+1 ms)Flushing stores of testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. at 1731369386882Flushing 6c54b823914e3bebb023069112e27160/testFamily: creating writer at 1731369386882Flushing 6c54b823914e3bebb023069112e27160/testFamily: appending metadata at 1731369386898 (+16 ms)Flushing 6c54b823914e3bebb023069112e27160/testFamily: closing flushed file at 1731369386898Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c668e98: reopening flushed file at 1731369387318 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6c54b823914e3bebb023069112e27160 in 447ms, sequenceid=8, compaction requested=false at 1731369387329 (+11 ms)Writing region close event to WAL at 1731369387329Running coprocessor post-close hooks at 1731369387334 (+5 ms)Closed at 1731369387334 2024-11-11T23:56:27,334 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1731369357739.6c54b823914e3bebb023069112e27160. 2024-11-11T23:56:27,334 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 479afff78def2884291e8d7249d59876, disabling compactions & flushes 2024-11-11T23:56:27,335 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. 2024-11-11T23:56:27,335 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. 2024-11-11T23:56:27,335 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. after waiting 0 ms 2024-11-11T23:56:27,335 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. 2024-11-11T23:56:27,335 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 479afff78def2884291e8d7249d59876 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-11T23:56:27,356 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScannerAsync/479afff78def2884291e8d7249d59876/.tmp/testFamily/17e7dedc51ac4e998e064e36bc7d30d0 is 61, key is row-0/testFamily:testQualifier/1731369381039/Put/seqid=0 2024-11-11T23:56:27,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741865_1041 (size=5049) 2024-11-11T23:56:27,496 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 479afff78def2884291e8d7249d59876, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:56:27,697 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 479afff78def2884291e8d7249d59876, 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:56:27,763 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScannerAsync/479afff78def2884291e8d7249d59876/.tmp/testFamily/17e7dedc51ac4e998e064e36bc7d30d0 2024-11-11T23:56:27,777 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScannerAsync/479afff78def2884291e8d7249d59876/.tmp/testFamily/17e7dedc51ac4e998e064e36bc7d30d0 as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScannerAsync/479afff78def2884291e8d7249d59876/testFamily/17e7dedc51ac4e998e064e36bc7d30d0 2024-11-11T23:56:27,786 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScannerAsync/479afff78def2884291e8d7249d59876/testFamily/17e7dedc51ac4e998e064e36bc7d30d0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-11T23:56:27,787 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 479afff78def2884291e8d7249d59876 in 452ms, sequenceid=8, compaction requested=false 2024-11-11T23:56:27,793 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/default/testNormalScanTimeoutOnOpenScannerAsync/479afff78def2884291e8d7249d59876/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-11T23:56:27,794 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. 2024-11-11T23:56:27,794 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 479afff78def2884291e8d7249d59876: Waiting for close lock at 1731369387334Running coprocessor pre-close hooks at 1731369387334Disabling compacts and flushes for region at 1731369387334Disabling writes for close at 1731369387335 (+1 ms)Obtaining lock to block concurrent updates at 1731369387335Preparing flush snapshotting stores in 479afff78def2884291e8d7249d59876 at 1731369387335Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731369387335Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. at 1731369387335Flushing 479afff78def2884291e8d7249d59876/testFamily: creating writer at 1731369387336 (+1 ms)Flushing 479afff78def2884291e8d7249d59876/testFamily: appending metadata at 1731369387355 (+19 ms)Flushing 479afff78def2884291e8d7249d59876/testFamily: closing flushed file at 1731369387355Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@115e7e5: reopening flushed file at 1731369387775 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 479afff78def2884291e8d7249d59876 in 452ms, sequenceid=8, compaction requested=false at 1731369387787 (+12 ms)Writing region close event to WAL at 1731369387788 (+1 ms)Running coprocessor post-close hooks at 1731369387794 (+6 ms)Closed at 1731369387794 2024-11-11T23:56:27,794 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731369377991.479afff78def2884291e8d7249d59876. 2024-11-11T23:56:27,795 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 981c9fbb3e7f4c993ecad2d02d339d83, disabling compactions & flushes 2024-11-11T23:56:27,795 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. 2024-11-11T23:56:27,795 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. 2024-11-11T23:56:27,795 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. after waiting 0 ms 2024-11-11T23:56:27,795 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. 2024-11-11T23:56:27,795 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 981c9fbb3e7f4c993ecad2d02d339d83 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-11T23:56:27,812 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNextAsync/981c9fbb3e7f4c993ecad2d02d339d83/.tmp/testFamily/4a9f0d495fe3497ab95a5b6527af53f5 is 61, key is row-0/testFamily:testQualifier/1731369336379/Put/seqid=0 2024-11-11T23:56:27,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741866_1042 (size=5049) 2024-11-11T23:56:27,897 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:56:28,097 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-11T23:56:28,097 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1325): Online Regions={981c9fbb3e7f4c993ecad2d02d339d83=hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83., 802a7072f042b432b2f57f46902e38c1=hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1.} 2024-11-11T23:56:28,098 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 802a7072f042b432b2f57f46902e38c1, 981c9fbb3e7f4c993ecad2d02d339d83 2024-11-11T23:56:28,219 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNextAsync/981c9fbb3e7f4c993ecad2d02d339d83/.tmp/testFamily/4a9f0d495fe3497ab95a5b6527af53f5 2024-11-11T23:56:28,232 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNextAsync/981c9fbb3e7f4c993ecad2d02d339d83/.tmp/testFamily/4a9f0d495fe3497ab95a5b6527af53f5 as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNextAsync/981c9fbb3e7f4c993ecad2d02d339d83/testFamily/4a9f0d495fe3497ab95a5b6527af53f5 2024-11-11T23:56:28,242 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNextAsync/981c9fbb3e7f4c993ecad2d02d339d83/testFamily/4a9f0d495fe3497ab95a5b6527af53f5, entries=4, sequenceid=8, filesize=4.9 K 2024-11-11T23:56:28,243 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 981c9fbb3e7f4c993ecad2d02d339d83 in 448ms, sequenceid=8, compaction requested=false 2024-11-11T23:56:28,249 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNextAsync/981c9fbb3e7f4c993ecad2d02d339d83/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-11T23:56:28,249 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. 2024-11-11T23:56:28,249 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 981c9fbb3e7f4c993ecad2d02d339d83: Waiting for close lock at 1731369387795Running coprocessor pre-close hooks at 1731369387795Disabling compacts and flushes for region at 1731369387795Disabling writes for close at 1731369387795Obtaining lock to block concurrent updates at 1731369387795Preparing flush snapshotting stores in 981c9fbb3e7f4c993ecad2d02d339d83 at 1731369387795Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731369387795Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. at 1731369387796 (+1 ms)Flushing 981c9fbb3e7f4c993ecad2d02d339d83/testFamily: creating writer at 1731369387796Flushing 981c9fbb3e7f4c993ecad2d02d339d83/testFamily: appending metadata at 1731369387811 (+15 ms)Flushing 981c9fbb3e7f4c993ecad2d02d339d83/testFamily: closing flushed file at 1731369387811Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77e852d7: reopening flushed file at 1731369388231 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 981c9fbb3e7f4c993ecad2d02d339d83 in 448ms, sequenceid=8, compaction requested=false at 1731369388243 (+12 ms)Writing region close event to WAL at 1731369388244 (+1 ms)Running coprocessor post-close hooks at 1731369388249 (+5 ms)Closed at 1731369388249 2024-11-11T23:56:28,250 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731369335345.981c9fbb3e7f4c993ecad2d02d339d83. 2024-11-11T23:56:28,250 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 802a7072f042b432b2f57f46902e38c1, disabling compactions & flushes 2024-11-11T23:56:28,250 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. 2024-11-11T23:56:28,250 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. 2024-11-11T23:56:28,250 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. after waiting 0 ms 2024-11-11T23:56:28,250 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. 2024-11-11T23:56:28,250 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 802a7072f042b432b2f57f46902e38c1 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-11T23:56:28,266 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNext/802a7072f042b432b2f57f46902e38c1/.tmp/testFamily/d613d34f26574c56b40e7b93aa382e3f is 61, key is row-0/testFamily:testQualifier/1731369371939/Put/seqid=0 2024-11-11T23:56:28,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741867_1043 (size=5049) 2024-11-11T23:56:28,298 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 802a7072f042b432b2f57f46902e38c1 2024-11-11T23:56:28,498 DEBUG [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1351): Waiting on 802a7072f042b432b2f57f46902e38c1 2024-11-11T23:56:28,672 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNext/802a7072f042b432b2f57f46902e38c1/.tmp/testFamily/d613d34f26574c56b40e7b93aa382e3f 2024-11-11T23:56:28,681 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNext/802a7072f042b432b2f57f46902e38c1/.tmp/testFamily/d613d34f26574c56b40e7b93aa382e3f as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNext/802a7072f042b432b2f57f46902e38c1/testFamily/d613d34f26574c56b40e7b93aa382e3f 2024-11-11T23:56:28,688 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNext/802a7072f042b432b2f57f46902e38c1/testFamily/d613d34f26574c56b40e7b93aa382e3f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-11T23:56:28,689 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 802a7072f042b432b2f57f46902e38c1 in 439ms, sequenceid=8, compaction requested=false 2024-11-11T23:56:28,695 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/data/hbase/testMetaScanTimeoutOnNext/802a7072f042b432b2f57f46902e38c1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-11T23:56:28,696 INFO [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. 2024-11-11T23:56:28,696 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 802a7072f042b432b2f57f46902e38c1: Waiting for close lock at 1731369388250Running coprocessor pre-close hooks at 1731369388250Disabling compacts and flushes for region at 1731369388250Disabling writes for close at 1731369388250Obtaining lock to block concurrent updates at 1731369388250Preparing flush snapshotting stores in 802a7072f042b432b2f57f46902e38c1 at 1731369388250Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731369388250Flushing stores of hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. at 1731369388251 (+1 ms)Flushing 802a7072f042b432b2f57f46902e38c1/testFamily: creating writer at 1731369388251Flushing 802a7072f042b432b2f57f46902e38c1/testFamily: appending metadata at 1731369388266 (+15 ms)Flushing 802a7072f042b432b2f57f46902e38c1/testFamily: closing flushed file at 1731369388266Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@346bd125: reopening flushed file at 1731369388680 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 802a7072f042b432b2f57f46902e38c1 in 439ms, sequenceid=8, compaction requested=false at 1731369388689 (+9 ms)Writing region close event to WAL at 1731369388690 (+1 ms)Running coprocessor post-close hooks at 1731369388696 (+6 ms)Closed at 1731369388696 2024-11-11T23:56:28,696 DEBUG [RS_CLOSE_REGION-regionserver/b70d29849faf:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1731369370911.802a7072f042b432b2f57f46902e38c1. 2024-11-11T23:56:28,699 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(976): stopping server b70d29849faf,41471,1731369291585; all regions closed. 2024-11-11T23:56:28,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741834_1010 (size=19479) 2024-11-11T23:56:28,710 DEBUG [RS:0;b70d29849faf:41471 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/oldWALs 2024-11-11T23:56:28,710 INFO [RS:0;b70d29849faf:41471 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b70d29849faf%2C41471%2C1731369291585.meta:.meta(num 1731369324667) 2024-11-11T23:56:28,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741833_1009 (size=20031) 2024-11-11T23:56:28,713 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/WALs/b70d29849faf,41471,1731369291585/b70d29849faf%2C41471%2C1731369291585.1731369324308 not finished, retry = 0 2024-11-11T23:56:28,819 DEBUG [RS:0;b70d29849faf:41471 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/oldWALs 2024-11-11T23:56:28,819 INFO [RS:0;b70d29849faf:41471 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b70d29849faf%2C41471%2C1731369291585:(num 1731369324308) 2024-11-11T23:56:28,819 DEBUG [RS:0;b70d29849faf:41471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T23:56:28,819 INFO [RS:0;b70d29849faf:41471 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T23:56:28,820 INFO [RS:0;b70d29849faf:41471 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T23:56:28,820 INFO [RS:0;b70d29849faf:41471 {}] hbase.ChoreService(370): Chore service for: regionserver/b70d29849faf:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T23:56:28,821 INFO [RS:0;b70d29849faf:41471 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T23:56:28,821 INFO [regionserver/b70d29849faf:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T23:56:28,823 INFO [RS:0;b70d29849faf:41471 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41471 2024-11-11T23:56:28,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b70d29849faf,41471,1731369291585 2024-11-11T23:56:28,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T23:56:28,827 INFO [RS:0;b70d29849faf:41471 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T23:56:28,828 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b70d29849faf,41471,1731369291585] 2024-11-11T23:56:28,829 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b70d29849faf,41471,1731369291585 already deleted, retry=false 2024-11-11T23:56:28,829 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b70d29849faf,41471,1731369291585 expired; onlineServers=0 2024-11-11T23:56:28,829 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b70d29849faf,34765,1731369291015' ***** 2024-11-11T23:56:28,829 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T23:56:28,829 INFO [M:0;b70d29849faf:34765 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T23:56:28,829 INFO [M:0;b70d29849faf:34765 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T23:56:28,829 DEBUG [M:0;b70d29849faf:34765 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T23:56:28,829 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T23:56:28,829 DEBUG [M:0;b70d29849faf:34765 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T23:56:28,829 DEBUG [master/b70d29849faf:0:becomeActiveMaster-HFileCleaner.large.0-1731369294124 {}] cleaner.HFileCleaner(306): Exit Thread[master/b70d29849faf:0:becomeActiveMaster-HFileCleaner.large.0-1731369294124,5,FailOnTimeoutGroup] 2024-11-11T23:56:28,829 DEBUG [master/b70d29849faf:0:becomeActiveMaster-HFileCleaner.small.0-1731369294125 {}] cleaner.HFileCleaner(306): Exit Thread[master/b70d29849faf:0:becomeActiveMaster-HFileCleaner.small.0-1731369294125,5,FailOnTimeoutGroup] 2024-11-11T23:56:28,830 INFO [M:0;b70d29849faf:34765 {}] hbase.ChoreService(370): Chore service for: master/b70d29849faf:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T23:56:28,830 INFO [M:0;b70d29849faf:34765 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T23:56:28,830 DEBUG [M:0;b70d29849faf:34765 {}] master.HMaster(1795): Stopping service threads 2024-11-11T23:56:28,830 INFO [M:0;b70d29849faf:34765 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T23:56:28,830 INFO [M:0;b70d29849faf:34765 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T23:56:28,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T23:56:28,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T23:56:28,831 INFO [M:0;b70d29849faf:34765 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T23:56:28,831 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T23:56:28,831 DEBUG [M:0;b70d29849faf:34765 {}] zookeeper.ZKUtil(347): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T23:56:28,831 WARN [M:0;b70d29849faf:34765 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T23:56:28,832 INFO [M:0;b70d29849faf:34765 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/.lastflushedseqids 2024-11-11T23:56:28,839 INFO [regionserver/b70d29849faf:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T23:56:28,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741868_1044 (size=469) 2024-11-11T23:56:28,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T23:56:28,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41471-0x1003136265b0001, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T23:56:28,929 INFO [RS:0;b70d29849faf:41471 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T23:56:28,929 INFO [RS:0;b70d29849faf:41471 {}] regionserver.HRegionServer(1031): Exiting; stopping=b70d29849faf,41471,1731369291585; zookeeper connection closed. 2024-11-11T23:56:28,930 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@607ae88 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@607ae88 2024-11-11T23:56:28,930 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T23:56:29,243 INFO [M:0;b70d29849faf:34765 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T23:56:29,244 INFO [M:0;b70d29849faf:34765 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T23:56:29,244 DEBUG [M:0;b70d29849faf:34765 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T23:56:29,244 INFO [M:0;b70d29849faf:34765 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:56:29,244 DEBUG [M:0;b70d29849faf:34765 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:56:29,244 DEBUG [M:0;b70d29849faf:34765 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T23:56:29,244 DEBUG [M:0;b70d29849faf:34765 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:56:29,244 INFO [M:0;b70d29849faf:34765 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.92 KB heapSize=190.39 KB 2024-11-11T23:56:29,261 DEBUG [M:0;b70d29849faf:34765 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4408239ca593491188df6fc55f7cb963 is 82, key is hbase:meta,,1/info:regioninfo/1731369324747/Put/seqid=0 2024-11-11T23:56:29,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741869_1045 (size=5672) 2024-11-11T23:56:29,666 INFO [M:0;b70d29849faf:34765 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4408239ca593491188df6fc55f7cb963 2024-11-11T23:56:29,691 DEBUG [M:0;b70d29849faf:34765 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd0479070eb14624af82325dc6ad3b97 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731369325916/Put/seqid=0 2024-11-11T23:56:29,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741870_1046 (size=14080) 2024-11-11T23:56:30,098 INFO [M:0;b70d29849faf:34765 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.37 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd0479070eb14624af82325dc6ad3b97 2024-11-11T23:56:30,110 INFO [M:0;b70d29849faf:34765 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cd0479070eb14624af82325dc6ad3b97 2024-11-11T23:56:30,125 DEBUG [M:0;b70d29849faf:34765 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a3b87bb3af16458c8edee012aa69385b is 69, key is b70d29849faf,41471,1731369291585/rs:state/1731369294164/Put/seqid=0 2024-11-11T23:56:30,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741871_1047 (size=5156) 2024-11-11T23:56:30,134 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T23:56:30,531 INFO [M:0;b70d29849faf:34765 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a3b87bb3af16458c8edee012aa69385b 2024-11-11T23:56:30,542 DEBUG [M:0;b70d29849faf:34765 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4408239ca593491188df6fc55f7cb963 as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4408239ca593491188df6fc55f7cb963 2024-11-11T23:56:30,548 INFO [M:0;b70d29849faf:34765 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4408239ca593491188df6fc55f7cb963, entries=8, sequenceid=311, filesize=5.5 K 2024-11-11T23:56:30,550 DEBUG [M:0;b70d29849faf:34765 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd0479070eb14624af82325dc6ad3b97 as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cd0479070eb14624af82325dc6ad3b97 2024-11-11T23:56:30,556 INFO [M:0;b70d29849faf:34765 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cd0479070eb14624af82325dc6ad3b97 2024-11-11T23:56:30,556 INFO [M:0;b70d29849faf:34765 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cd0479070eb14624af82325dc6ad3b97, entries=33, sequenceid=311, filesize=13.8 K 2024-11-11T23:56:30,557 DEBUG [M:0;b70d29849faf:34765 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a3b87bb3af16458c8edee012aa69385b as hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a3b87bb3af16458c8edee012aa69385b 2024-11-11T23:56:30,563 INFO [M:0;b70d29849faf:34765 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39869/user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a3b87bb3af16458c8edee012aa69385b, entries=1, sequenceid=311, filesize=5.0 K 2024-11-11T23:56:30,564 INFO [M:0;b70d29849faf:34765 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.92 KB/166831, heapSize ~190.09 KB/194656, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1320ms, sequenceid=311, compaction requested=false 2024-11-11T23:56:30,565 INFO [M:0;b70d29849faf:34765 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T23:56:30,565 DEBUG [M:0;b70d29849faf:34765 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731369389244Disabling compacts and flushes for region at 1731369389244Disabling writes for close at 1731369389244Obtaining lock to block concurrent updates at 1731369389244Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731369389244Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166831, getHeapSize=194896, getOffHeapSize=0, getCellsCount=358 at 1731369389245 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731369389245Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731369389245Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731369389260 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731369389260Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731369389673 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731369389691 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731369389691Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731369390110 (+419 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731369390124 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731369390124Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2882ee53: reopening flushed file at 1731369390541 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53132f74: reopening flushed file at 1731369390549 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2178a3f7: reopening flushed file at 1731369390556 (+7 ms)Finished flush of dataSize ~162.92 KB/166831, heapSize ~190.09 KB/194656, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1320ms, sequenceid=311, compaction requested=false at 1731369390564 (+8 ms)Writing region close event to WAL at 1731369390565 (+1 ms)Closed at 1731369390565 2024-11-11T23:56:30,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37981 is added to blk_1073741830_1006 (size=187653) 2024-11-11T23:56:30,568 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/1e85d791-62d4-f39f-99f5-d71ff9180018/MasterData/WALs/b70d29849faf,34765,1731369291015/b70d29849faf%2C34765%2C1731369291015.1731369293643 not finished, retry = 0 2024-11-11T23:56:30,670 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T23:56:30,670 INFO [M:0;b70d29849faf:34765 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T23:56:30,671 INFO [M:0;b70d29849faf:34765 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34765 2024-11-11T23:56:30,671 INFO [M:0;b70d29849faf:34765 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T23:56:30,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T23:56:30,773 INFO [M:0;b70d29849faf:34765 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T23:56:30,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34765-0x1003136265b0000, quorum=127.0.0.1:56838, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T23:56:30,778 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@429b6cf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T23:56:30,780 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T23:56:30,780 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T23:56:30,780 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T23:56:30,780 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/hadoop.log.dir/,STOPPED} 2024-11-11T23:56:30,783 WARN [BP-716215489-172.17.0.2-1731369288393 heartbeating to localhost/127.0.0.1:39869 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T23:56:30,783 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T23:56:30,783 WARN [BP-716215489-172.17.0.2-1731369288393 heartbeating to localhost/127.0.0.1:39869 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-716215489-172.17.0.2-1731369288393 (Datanode Uuid 58e05cbd-3d48-42e0-a8b2-a664a3b734ee) service to localhost/127.0.0.1:39869 2024-11-11T23:56:30,783 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T23:56:30,784 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/cluster_2dda0b1b-20c8-cfdb-81b8-35b1b1adfbd5/data/data1/current/BP-716215489-172.17.0.2-1731369288393 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T23:56:30,784 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/cluster_2dda0b1b-20c8-cfdb-81b8-35b1b1adfbd5/data/data2/current/BP-716215489-172.17.0.2-1731369288393 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T23:56:30,784 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T23:56:30,792 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@40b93e99{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T23:56:30,792 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T23:56:30,792 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T23:56:30,793 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T23:56:30,793 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/90a5f288-e745-592e-0dc2-52dd0fcdf1a1/hadoop.log.dir/,STOPPED} 2024-11-11T23:56:30,800 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T23:56:30,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down