2024-12-03 22:58:03,929 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-03 22:58:03,940 main DEBUG Took 0.009445 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 22:58:03,941 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 22:58:03,941 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 22:58:03,942 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 22:58:03,943 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 22:58:03,954 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 22:58:03,966 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 22:58:03,968 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 22:58:03,968 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 22:58:03,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 22:58:03,969 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 22:58:03,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 22:58:03,971 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 22:58:03,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 22:58:03,972 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 22:58:03,972 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 22:58:03,973 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 22:58:03,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 22:58:03,974 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 22:58:03,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 22:58:03,975 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 22:58:03,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 22:58:03,976 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 22:58:03,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 22:58:03,977 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 22:58:03,977 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 22:58:03,978 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 22:58:03,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 22:58:03,978 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 22:58:03,979 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 22:58:03,979 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 22:58:03,980 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 22:58:03,982 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 22:58:03,983 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 22:58:03,985 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 22:58:03,986 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 22:58:03,988 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 22:58:03,988 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 22:58:03,996 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 22:58:03,999 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 22:58:04,002 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 22:58:04,002 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 22:58:04,003 main DEBUG createAppenders(={Console}) 2024-12-03 22:58:04,004 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-03 22:58:04,004 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-03 22:58:04,005 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-03 22:58:04,005 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 22:58:04,006 main DEBUG OutputStream closed 2024-12-03 22:58:04,006 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 22:58:04,006 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 22:58:04,007 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-03 22:58:04,086 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 22:58:04,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 22:58:04,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 22:58:04,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 22:58:04,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 22:58:04,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 22:58:04,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 22:58:04,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 22:58:04,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 22:58:04,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 22:58:04,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 22:58:04,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 22:58:04,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 22:58:04,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 22:58:04,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 22:58:04,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 22:58:04,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 22:58:04,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 22:58:04,099 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 22:58:04,099 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-03 22:58:04,099 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 22:58:04,100 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-03T22:58:04,115 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-03 22:58:04,118 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 22:58:04,119 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T22:58:04,341 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18 2024-12-03T22:58:04,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T22:58:04,385 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/cluster_a42b0e19-4ea0-5ed7-aabb-16b1ec3e8836, deleteOnExit=true 2024-12-03T22:58:04,385 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T22:58:04,387 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/test.cache.data in system properties and HBase conf 2024-12-03T22:58:04,387 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T22:58:04,388 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/hadoop.log.dir in system properties and HBase conf 2024-12-03T22:58:04,389 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T22:58:04,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T22:58:04,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T22:58:04,506 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T22:58:04,622 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T22:58:04,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T22:58:04,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T22:58:04,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T22:58:04,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T22:58:04,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T22:58:04,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T22:58:04,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T22:58:04,634 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T22:58:04,635 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T22:58:04,635 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/nfs.dump.dir in system properties and HBase conf 2024-12-03T22:58:04,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/java.io.tmpdir in system properties and HBase conf 2024-12-03T22:58:04,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T22:58:04,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T22:58:04,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T22:58:05,660 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T22:58:05,721 INFO [Time-limited test {}] log.Log(170): Logging initialized @2448ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T22:58:05,779 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T22:58:05,828 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T22:58:05,845 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T22:58:05,846 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T22:58:05,847 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T22:58:05,858 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T22:58:05,860 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/hadoop.log.dir/,AVAILABLE} 2024-12-03T22:58:05,862 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T22:58:06,031 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/java.io.tmpdir/jetty-localhost-34643-hadoop-hdfs-3_4_1-tests_jar-_-any-12134853716505398176/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T22:58:06,037 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:34643} 2024-12-03T22:58:06,037 INFO [Time-limited test {}] server.Server(415): Started @2765ms 2024-12-03T22:58:06,508 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T22:58:06,514 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T22:58:06,515 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T22:58:06,515 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T22:58:06,515 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T22:58:06,516 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/hadoop.log.dir/,AVAILABLE} 2024-12-03T22:58:06,517 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T22:58:06,610 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/java.io.tmpdir/jetty-localhost-37105-hadoop-hdfs-3_4_1-tests_jar-_-any-2729943244548805691/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T22:58:06,611 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:37105} 2024-12-03T22:58:06,611 INFO [Time-limited test {}] server.Server(415): Started @3339ms 2024-12-03T22:58:06,656 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T22:58:07,415 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/cluster_a42b0e19-4ea0-5ed7-aabb-16b1ec3e8836/data/data1/current/BP-1108234711-172.17.0.2-1733266685190/current, will proceed with Du for space computation calculation, 2024-12-03T22:58:07,415 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/cluster_a42b0e19-4ea0-5ed7-aabb-16b1ec3e8836/data/data2/current/BP-1108234711-172.17.0.2-1733266685190/current, will proceed with Du for space computation calculation, 2024-12-03T22:58:07,439 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T22:58:07,504 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x771efa4463e610bc with lease ID 0xe9be05bd824151dd: Processing first storage report for DS-431c436b-072d-4b43-8952-afae962de200 from datanode DatanodeRegistration(127.0.0.1:36673, datanodeUuid=ee7209ab-e464-43bb-9153-701373c951ef, infoPort=46385, infoSecurePort=0, ipcPort=35479, storageInfo=lv=-57;cid=testClusterID;nsid=596120931;c=1733266685190) 2024-12-03T22:58:07,505 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x771efa4463e610bc with lease ID 0xe9be05bd824151dd: from storage DS-431c436b-072d-4b43-8952-afae962de200 node DatanodeRegistration(127.0.0.1:36673, datanodeUuid=ee7209ab-e464-43bb-9153-701373c951ef, infoPort=46385, infoSecurePort=0, ipcPort=35479, storageInfo=lv=-57;cid=testClusterID;nsid=596120931;c=1733266685190), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-03T22:58:07,506 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x771efa4463e610bc with lease ID 0xe9be05bd824151dd: Processing first storage report for DS-1033d35d-cbe8-4fc3-a02c-1ca6cda9e2e0 from datanode DatanodeRegistration(127.0.0.1:36673, datanodeUuid=ee7209ab-e464-43bb-9153-701373c951ef, infoPort=46385, infoSecurePort=0, ipcPort=35479, storageInfo=lv=-57;cid=testClusterID;nsid=596120931;c=1733266685190) 2024-12-03T22:58:07,506 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x771efa4463e610bc with lease ID 0xe9be05bd824151dd: from storage DS-1033d35d-cbe8-4fc3-a02c-1ca6cda9e2e0 node DatanodeRegistration(127.0.0.1:36673, datanodeUuid=ee7209ab-e464-43bb-9153-701373c951ef, infoPort=46385, infoSecurePort=0, ipcPort=35479, storageInfo=lv=-57;cid=testClusterID;nsid=596120931;c=1733266685190), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T22:58:07,541 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18 2024-12-03T22:58:07,666 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/cluster_a42b0e19-4ea0-5ed7-aabb-16b1ec3e8836/zookeeper_0, clientPort=55134, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/cluster_a42b0e19-4ea0-5ed7-aabb-16b1ec3e8836/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/cluster_a42b0e19-4ea0-5ed7-aabb-16b1ec3e8836/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T22:58:07,684 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55134 2024-12-03T22:58:07,705 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T22:58:07,710 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T22:58:08,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741825_1001 (size=7) 2024-12-03T22:58:08,440 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58 with version=8 2024-12-03T22:58:08,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/hbase-staging 2024-12-03T22:58:08,509 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T22:58:08,734 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ed8014902436:0 server-side Connection retries=9 2024-12-03T22:58:08,745 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T22:58:08,746 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T22:58:08,752 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T22:58:08,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T22:58:08,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T22:58:08,883 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T22:58:08,933 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T22:58:08,940 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T22:58:08,943 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T22:58:08,963 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 114218 (auto-detected) 2024-12-03T22:58:08,964 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-03T22:58:08,980 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44997 2024-12-03T22:58:08,998 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44997 connecting to ZooKeeper ensemble=127.0.0.1:55134 2024-12-03T22:58:09,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:449970x0, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T22:58:09,083 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44997-0x1019eb459510000 connected 2024-12-03T22:58:09,178 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T22:58:09,182 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T22:58:09,193 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T22:58:09,197 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58, hbase.cluster.distributed=false 2024-12-03T22:58:09,223 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T22:58:09,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44997 2024-12-03T22:58:09,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44997 2024-12-03T22:58:09,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44997 2024-12-03T22:58:09,230 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44997 2024-12-03T22:58:09,230 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44997 2024-12-03T22:58:09,341 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ed8014902436:0 server-side Connection retries=9 2024-12-03T22:58:09,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T22:58:09,344 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T22:58:09,344 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T22:58:09,344 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T22:58:09,344 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T22:58:09,347 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T22:58:09,349 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T22:58:09,349 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36731 2024-12-03T22:58:09,351 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36731 connecting to ZooKeeper ensemble=127.0.0.1:55134 2024-12-03T22:58:09,352 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T22:58:09,355 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T22:58:09,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:367310x0, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T22:58:09,388 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:367310x0, quorum=127.0.0.1:55134, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T22:58:09,388 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36731-0x1019eb459510001 connected 2024-12-03T22:58:09,391 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T22:58:09,399 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T22:58:09,401 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T22:58:09,406 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T22:58:09,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36731 2024-12-03T22:58:09,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36731 2024-12-03T22:58:09,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36731 2024-12-03T22:58:09,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36731 2024-12-03T22:58:09,409 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36731 2024-12-03T22:58:09,424 DEBUG [M:0;ed8014902436:44997 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ed8014902436:44997 2024-12-03T22:58:09,425 INFO [master/ed8014902436:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ed8014902436,44997,1733266688559 2024-12-03T22:58:09,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T22:58:09,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T22:58:09,439 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ed8014902436,44997,1733266688559 2024-12-03T22:58:09,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:58:09,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T22:58:09,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:58:09,467 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T22:58:09,468 INFO [master/ed8014902436:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ed8014902436,44997,1733266688559 from backup master directory 2024-12-03T22:58:09,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T22:58:09,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ed8014902436,44997,1733266688559 2024-12-03T22:58:09,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T22:58:09,479 WARN [master/ed8014902436:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T22:58:09,480 INFO [master/ed8014902436:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ed8014902436,44997,1733266688559 2024-12-03T22:58:09,482 INFO [master/ed8014902436:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T22:58:09,483 INFO [master/ed8014902436:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T22:58:09,532 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/hbase.id] with ID: 8c57f45c-be26-4ca2-b237-0e73ddd235fe 2024-12-03T22:58:09,532 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/.tmp/hbase.id 2024-12-03T22:58:09,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741826_1002 (size=42) 2024-12-03T22:58:09,943 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/.tmp/hbase.id]:[hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/hbase.id] 2024-12-03T22:58:09,984 INFO [master/ed8014902436:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T22:58:09,989 INFO [master/ed8014902436:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T22:58:10,008 INFO [master/ed8014902436:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-03T22:58:10,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:58:10,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:58:10,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741827_1003 (size=196) 2024-12-03T22:58:10,057 INFO [master/ed8014902436:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T22:58:10,060 INFO [master/ed8014902436:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T22:58:10,080 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T22:58:10,085 INFO [master/ed8014902436:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T22:58:10,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741828_1004 (size=1189) 2024-12-03T22:58:10,545 INFO [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store 2024-12-03T22:58:10,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741829_1005 (size=34) 2024-12-03T22:58:10,971 INFO [master/ed8014902436:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T22:58:10,975 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:58:10,976 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T22:58:10,976 INFO [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T22:58:10,977 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T22:58:10,978 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T22:58:10,978 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T22:58:10,978 INFO [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T22:58:10,979 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733266690976Disabling compacts and flushes for region at 1733266690976Disabling writes for close at 1733266690978 (+2 ms)Writing region close event to WAL at 1733266690978Closed at 1733266690978 2024-12-03T22:58:10,981 WARN [master/ed8014902436:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/.initializing 2024-12-03T22:58:10,981 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/WALs/ed8014902436,44997,1733266688559 2024-12-03T22:58:10,989 INFO [master/ed8014902436:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T22:58:11,003 INFO [master/ed8014902436:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ed8014902436%2C44997%2C1733266688559, suffix=, logDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/WALs/ed8014902436,44997,1733266688559, archiveDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/oldWALs, maxLogs=10 2024-12-03T22:58:11,023 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/WALs/ed8014902436,44997,1733266688559/ed8014902436%2C44997%2C1733266688559.1733266691007, exclude list is [], retry=0 2024-12-03T22:58:11,037 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36673,DS-431c436b-072d-4b43-8952-afae962de200,DISK] 2024-12-03T22:58:11,040 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T22:58:11,070 INFO [master/ed8014902436:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/WALs/ed8014902436,44997,1733266688559/ed8014902436%2C44997%2C1733266688559.1733266691007 2024-12-03T22:58:11,071 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46385:46385)] 2024-12-03T22:58:11,072 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T22:58:11,072 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:58:11,075 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T22:58:11,075 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T22:58:11,109 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T22:58:11,130 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T22:58:11,133 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:11,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T22:58:11,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T22:58:11,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T22:58:11,139 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:11,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T22:58:11,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T22:58:11,142 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T22:58:11,142 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:11,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T22:58:11,144 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T22:58:11,146 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T22:58:11,146 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:11,147 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T22:58:11,147 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T22:58:11,151 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T22:58:11,152 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T22:58:11,157 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T22:58:11,157 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T22:58:11,161 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T22:58:11,165 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T22:58:11,170 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T22:58:11,171 INFO [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72041912, jitterRate=0.07350814342498779}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T22:58:11,177 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733266691086Initializing all the Stores at 1733266691088 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733266691089 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266691089Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266691090 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266691090Cleaning up temporary data from old regions at 1733266691158 (+68 ms)Region opened successfully at 1733266691177 (+19 ms) 2024-12-03T22:58:11,178 INFO [master/ed8014902436:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T22:58:11,208 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@662cbb5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ed8014902436/172.17.0.2:0 2024-12-03T22:58:11,235 INFO [master/ed8014902436:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T22:58:11,244 INFO [master/ed8014902436:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T22:58:11,245 INFO [master/ed8014902436:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T22:58:11,247 INFO [master/ed8014902436:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T22:58:11,248 INFO [master/ed8014902436:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T22:58:11,252 INFO [master/ed8014902436:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-03T22:58:11,252 INFO [master/ed8014902436:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T22:58:11,274 INFO [master/ed8014902436:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T22:58:11,282 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T22:58:11,315 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T22:58:11,318 INFO [master/ed8014902436:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T22:58:11,320 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T22:58:11,332 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T22:58:11,335 INFO [master/ed8014902436:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T22:58:11,340 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T22:58:11,348 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T22:58:11,350 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T22:58:11,357 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T22:58:11,375 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T22:58:11,386 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T22:58:11,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T22:58:11,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T22:58:11,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:58:11,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:58:11,402 INFO [master/ed8014902436:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ed8014902436,44997,1733266688559, sessionid=0x1019eb459510000, setting cluster-up flag (Was=false) 2024-12-03T22:58:11,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:58:11,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:58:11,462 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T22:58:11,467 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ed8014902436,44997,1733266688559 2024-12-03T22:58:11,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:58:11,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:58:11,515 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T22:58:11,519 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ed8014902436,44997,1733266688559 2024-12-03T22:58:11,528 INFO [master/ed8014902436:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T22:58:11,586 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T22:58:11,593 INFO [master/ed8014902436:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T22:58:11,599 INFO [master/ed8014902436:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T22:58:11,604 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ed8014902436,44997,1733266688559 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T22:58:11,610 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ed8014902436:0, corePoolSize=5, maxPoolSize=5 2024-12-03T22:58:11,610 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ed8014902436:0, corePoolSize=5, maxPoolSize=5 2024-12-03T22:58:11,610 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ed8014902436:0, corePoolSize=5, maxPoolSize=5 2024-12-03T22:58:11,610 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ed8014902436:0, corePoolSize=5, maxPoolSize=5 2024-12-03T22:58:11,610 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ed8014902436:0, corePoolSize=10, maxPoolSize=10 2024-12-03T22:58:11,610 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ed8014902436:0, corePoolSize=1, maxPoolSize=1 2024-12-03T22:58:11,611 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ed8014902436:0, corePoolSize=2, maxPoolSize=2 2024-12-03T22:58:11,611 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ed8014902436:0, corePoolSize=1, maxPoolSize=1 2024-12-03T22:58:11,612 INFO [master/ed8014902436:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733266721612 2024-12-03T22:58:11,613 INFO [master/ed8014902436:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T22:58:11,614 INFO [master/ed8014902436:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T22:58:11,615 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(746): ClusterId : 8c57f45c-be26-4ca2-b237-0e73ddd235fe 2024-12-03T22:58:11,616 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T22:58:11,616 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T22:58:11,617 INFO [master/ed8014902436:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T22:58:11,617 DEBUG [RS:0;ed8014902436:36731 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T22:58:11,618 INFO [master/ed8014902436:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T22:58:11,618 INFO [master/ed8014902436:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T22:58:11,618 INFO [master/ed8014902436:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T22:58:11,619 INFO [master/ed8014902436:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:11,622 INFO [master/ed8014902436:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T22:58:11,623 INFO [master/ed8014902436:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T22:58:11,623 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:11,623 INFO [master/ed8014902436:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T22:58:11,623 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T22:58:11,624 INFO [master/ed8014902436:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T22:58:11,625 INFO [master/ed8014902436:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T22:58:11,626 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ed8014902436:0:becomeActiveMaster-HFileCleaner.large.0-1733266691626,5,FailOnTimeoutGroup] 2024-12-03T22:58:11,627 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ed8014902436:0:becomeActiveMaster-HFileCleaner.small.0-1733266691626,5,FailOnTimeoutGroup] 2024-12-03T22:58:11,627 INFO [master/ed8014902436:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:11,627 INFO [master/ed8014902436:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T22:58:11,628 INFO [master/ed8014902436:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:11,628 INFO [master/ed8014902436:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:11,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741831_1007 (size=1321) 2024-12-03T22:58:11,651 DEBUG [RS:0;ed8014902436:36731 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T22:58:11,651 DEBUG [RS:0;ed8014902436:36731 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T22:58:11,664 DEBUG [RS:0;ed8014902436:36731 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T22:58:11,664 DEBUG [RS:0;ed8014902436:36731 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@82c47fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ed8014902436/172.17.0.2:0 2024-12-03T22:58:11,683 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ed8014902436:36731 2024-12-03T22:58:11,686 INFO [RS:0;ed8014902436:36731 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T22:58:11,687 INFO [RS:0;ed8014902436:36731 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T22:58:11,687 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T22:58:11,689 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(2659): reportForDuty to master=ed8014902436,44997,1733266688559 with port=36731, startcode=1733266689308 2024-12-03T22:58:11,699 DEBUG [RS:0;ed8014902436:36731 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T22:58:11,724 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43053, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T22:58:11,729 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44997 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ed8014902436,36731,1733266689308 2024-12-03T22:58:11,731 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44997 {}] master.ServerManager(517): Registering regionserver=ed8014902436,36731,1733266689308 2024-12-03T22:58:11,742 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58 2024-12-03T22:58:11,742 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41127 2024-12-03T22:58:11,742 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T22:58:11,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T22:58:11,762 DEBUG [RS:0;ed8014902436:36731 {}] zookeeper.ZKUtil(111): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ed8014902436,36731,1733266689308 2024-12-03T22:58:11,763 WARN [RS:0;ed8014902436:36731 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T22:58:11,763 INFO [RS:0;ed8014902436:36731 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T22:58:11,763 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/WALs/ed8014902436,36731,1733266689308 2024-12-03T22:58:11,765 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ed8014902436,36731,1733266689308] 2024-12-03T22:58:11,788 INFO [RS:0;ed8014902436:36731 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T22:58:11,799 INFO [RS:0;ed8014902436:36731 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T22:58:11,803 INFO [RS:0;ed8014902436:36731 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T22:58:11,803 INFO [RS:0;ed8014902436:36731 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:11,804 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T22:58:11,808 INFO [RS:0;ed8014902436:36731 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T22:58:11,809 INFO [RS:0;ed8014902436:36731 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:11,809 DEBUG [RS:0;ed8014902436:36731 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ed8014902436:0, corePoolSize=1, maxPoolSize=1 2024-12-03T22:58:11,810 DEBUG [RS:0;ed8014902436:36731 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ed8014902436:0, corePoolSize=1, maxPoolSize=1 2024-12-03T22:58:11,810 DEBUG [RS:0;ed8014902436:36731 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0, corePoolSize=1, maxPoolSize=1 2024-12-03T22:58:11,810 DEBUG [RS:0;ed8014902436:36731 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ed8014902436:0, corePoolSize=1, maxPoolSize=1 2024-12-03T22:58:11,810 DEBUG [RS:0;ed8014902436:36731 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ed8014902436:0, corePoolSize=1, maxPoolSize=1 2024-12-03T22:58:11,810 DEBUG [RS:0;ed8014902436:36731 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ed8014902436:0, corePoolSize=2, maxPoolSize=2 2024-12-03T22:58:11,810 DEBUG [RS:0;ed8014902436:36731 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ed8014902436:0, corePoolSize=1, maxPoolSize=1 2024-12-03T22:58:11,810 DEBUG [RS:0;ed8014902436:36731 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ed8014902436:0, corePoolSize=1, maxPoolSize=1 2024-12-03T22:58:11,811 DEBUG [RS:0;ed8014902436:36731 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ed8014902436:0, corePoolSize=1, maxPoolSize=1 2024-12-03T22:58:11,811 DEBUG [RS:0;ed8014902436:36731 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ed8014902436:0, corePoolSize=1, maxPoolSize=1 2024-12-03T22:58:11,811 DEBUG [RS:0;ed8014902436:36731 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ed8014902436:0, corePoolSize=1, maxPoolSize=1 2024-12-03T22:58:11,811 DEBUG [RS:0;ed8014902436:36731 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ed8014902436:0, corePoolSize=1, maxPoolSize=1 2024-12-03T22:58:11,811 DEBUG [RS:0;ed8014902436:36731 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ed8014902436:0, corePoolSize=3, maxPoolSize=3 2024-12-03T22:58:11,811 DEBUG [RS:0;ed8014902436:36731 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ed8014902436:0, corePoolSize=3, maxPoolSize=3 2024-12-03T22:58:11,812 INFO [RS:0;ed8014902436:36731 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:11,812 INFO [RS:0;ed8014902436:36731 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:11,812 INFO [RS:0;ed8014902436:36731 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:11,812 INFO [RS:0;ed8014902436:36731 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:11,813 INFO [RS:0;ed8014902436:36731 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:11,813 INFO [RS:0;ed8014902436:36731 {}] hbase.ChoreService(168): Chore ScheduledChore name=ed8014902436,36731,1733266689308-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T22:58:11,828 INFO [RS:0;ed8014902436:36731 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T22:58:11,829 INFO [RS:0;ed8014902436:36731 {}] hbase.ChoreService(168): Chore ScheduledChore name=ed8014902436,36731,1733266689308-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:11,829 INFO [RS:0;ed8014902436:36731 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:11,829 INFO [RS:0;ed8014902436:36731 {}] regionserver.Replication(171): ed8014902436,36731,1733266689308 started 2024-12-03T22:58:11,844 INFO [RS:0;ed8014902436:36731 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:11,844 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1482): Serving as ed8014902436,36731,1733266689308, RpcServer on ed8014902436/172.17.0.2:36731, sessionid=0x1019eb459510001 2024-12-03T22:58:11,845 DEBUG [RS:0;ed8014902436:36731 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T22:58:11,845 DEBUG [RS:0;ed8014902436:36731 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ed8014902436,36731,1733266689308 2024-12-03T22:58:11,845 DEBUG [RS:0;ed8014902436:36731 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ed8014902436,36731,1733266689308' 2024-12-03T22:58:11,845 DEBUG [RS:0;ed8014902436:36731 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T22:58:11,846 DEBUG [RS:0;ed8014902436:36731 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T22:58:11,847 DEBUG [RS:0;ed8014902436:36731 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T22:58:11,847 DEBUG [RS:0;ed8014902436:36731 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T22:58:11,847 DEBUG [RS:0;ed8014902436:36731 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ed8014902436,36731,1733266689308 2024-12-03T22:58:11,847 DEBUG [RS:0;ed8014902436:36731 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ed8014902436,36731,1733266689308' 2024-12-03T22:58:11,847 DEBUG [RS:0;ed8014902436:36731 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T22:58:11,847 DEBUG [RS:0;ed8014902436:36731 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T22:58:11,848 DEBUG [RS:0;ed8014902436:36731 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T22:58:11,848 INFO [RS:0;ed8014902436:36731 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T22:58:11,848 INFO [RS:0;ed8014902436:36731 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T22:58:12,038 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T22:58:12,040 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58 2024-12-03T22:58:12,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741832_1008 (size=32) 2024-12-03T22:58:12,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:58:12,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T22:58:12,480 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T22:58:12,480 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:12,482 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T22:58:12,482 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T22:58:12,490 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T22:58:12,490 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:12,492 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T22:58:12,492 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T22:58:12,502 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T22:58:12,503 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:12,504 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T22:58:12,504 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T22:58:12,509 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T22:58:12,509 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:12,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T22:58:12,511 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T22:58:12,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740 2024-12-03T22:58:12,514 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740 2024-12-03T22:58:12,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T22:58:12,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T22:58:12,520 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T22:58:12,525 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T22:58:12,559 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T22:58:12,561 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73407896, jitterRate=0.09386289119720459}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T22:58:12,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733266692453Initializing all the Stores at 1733266692457 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733266692457Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733266692466 (+9 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266692466Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733266692466Cleaning up temporary data from old regions at 1733266692519 (+53 ms)Region opened successfully at 1733266692563 (+44 ms) 2024-12-03T22:58:12,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T22:58:12,564 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T22:58:12,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T22:58:12,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T22:58:12,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T22:58:12,581 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T22:58:12,581 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733266692564Disabling compacts and flushes for region at 1733266692564Disabling writes for close at 1733266692564Writing region close event to WAL at 1733266692580 (+16 ms)Closed at 1733266692581 (+1 ms) 2024-12-03T22:58:12,586 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T22:58:12,586 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T22:58:12,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T22:58:12,640 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T22:58:12,644 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T22:58:12,795 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:13,796 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:14,797 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:15,798 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:16,799 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:17,800 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:18,801 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:18,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T22:58:18,931 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T22:58:19,802 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:20,803 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:21,804 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:22,805 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:23,806 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:24,807 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:25,808 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:26,809 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:27,810 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:28,811 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:29,811 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:30,812 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:31,813 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:32,813 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:33,814 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:34,815 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:35,816 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:36,817 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:37,538 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T22:58:37,817 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:38,818 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:39,819 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:40,820 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:41,821 WARN [ed8014902436:44997 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T22:58:41,860 INFO [RS:0;ed8014902436:36731 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T22:58:41,865 INFO [RS:0;ed8014902436:36731 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ed8014902436%2C36731%2C1733266689308, suffix=, logDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/WALs/ed8014902436,36731,1733266689308, archiveDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/oldWALs, maxLogs=32 2024-12-03T22:58:41,878 DEBUG [RS:0;ed8014902436:36731 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/WALs/ed8014902436,36731,1733266689308/ed8014902436%2C36731%2C1733266689308.1733266721867, exclude list is [], retry=0 2024-12-03T22:58:41,882 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36673,DS-431c436b-072d-4b43-8952-afae962de200,DISK] 2024-12-03T22:58:41,885 INFO [RS:0;ed8014902436:36731 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/WALs/ed8014902436,36731,1733266689308/ed8014902436%2C36731%2C1733266689308.1733266721867 2024-12-03T22:58:41,885 DEBUG [RS:0;ed8014902436:36731 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46385:46385)] 2024-12-03T22:58:42,074 DEBUG [ed8014902436:44997 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T22:58:42,087 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:58:42,092 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ed8014902436,36731,1733266689308, state=OPENING 2024-12-03T22:58:42,125 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T22:58:42,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:58:42,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:58:42,135 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T22:58:42,136 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T22:58:42,139 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T22:58:42,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ed8014902436,36731,1733266689308}] 2024-12-03T22:58:42,322 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T22:58:42,325 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51295, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T22:58:42,337 INFO [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T22:58:42,338 INFO [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T22:58:42,338 INFO [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T22:58:42,342 INFO [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ed8014902436%2C36731%2C1733266689308.meta, suffix=.meta, logDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/WALs/ed8014902436,36731,1733266689308, archiveDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/oldWALs, maxLogs=32 2024-12-03T22:58:42,362 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/WALs/ed8014902436,36731,1733266689308/ed8014902436%2C36731%2C1733266689308.meta.1733266722344.meta, exclude list is [], retry=0 2024-12-03T22:58:42,365 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36673,DS-431c436b-072d-4b43-8952-afae962de200,DISK] 2024-12-03T22:58:42,368 INFO [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/WALs/ed8014902436,36731,1733266689308/ed8014902436%2C36731%2C1733266689308.meta.1733266722344.meta 2024-12-03T22:58:42,369 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46385:46385)] 2024-12-03T22:58:42,369 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T22:58:42,371 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T22:58:42,373 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T22:58:42,377 INFO [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T22:58:42,381 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T22:58:42,381 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:58:42,382 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T22:58:42,382 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T22:58:42,386 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T22:58:42,387 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T22:58:42,387 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:42,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T22:58:42,389 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T22:58:42,391 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T22:58:42,391 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:42,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T22:58:42,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T22:58:42,393 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T22:58:42,393 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:42,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T22:58:42,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T22:58:42,396 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T22:58:42,396 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:42,397 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T22:58:42,397 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T22:58:42,399 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740 2024-12-03T22:58:42,402 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740 2024-12-03T22:58:42,405 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T22:58:42,405 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T22:58:42,406 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T22:58:42,409 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T22:58:42,410 INFO [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67947877, jitterRate=0.012502267956733704}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T22:58:42,411 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T22:58:42,412 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733266722383Writing region info on filesystem at 1733266722383Initializing all the Stores at 1733266722385 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733266722385Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733266722386 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266722386Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733266722386Cleaning up temporary data from old regions at 1733266722405 (+19 ms)Running coprocessor post-open hooks at 1733266722411 (+6 ms)Region opened successfully at 1733266722412 (+1 ms) 2024-12-03T22:58:42,418 INFO [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733266722311 2024-12-03T22:58:42,429 DEBUG [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T22:58:42,429 INFO [RS_OPEN_META-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T22:58:42,431 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:58:42,434 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ed8014902436,36731,1733266689308, state=OPEN 2024-12-03T22:58:42,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T22:58:42,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T22:58:42,481 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T22:58:42,481 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T22:58:42,482 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ed8014902436,36731,1733266689308 2024-12-03T22:58:42,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T22:58:42,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ed8014902436,36731,1733266689308 in 340 msec 2024-12-03T22:58:42,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T22:58:42,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.8960 sec 2024-12-03T22:58:42,498 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T22:58:42,499 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T22:58:42,514 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T22:58:42,515 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ed8014902436,36731,1733266689308, seqNum=-1] 2024-12-03T22:58:42,532 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T22:58:42,534 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56419, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T22:58:42,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.9970 sec 2024-12-03T22:58:42,551 INFO [master/ed8014902436:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733266722551, completionTime=-1 2024-12-03T22:58:42,554 INFO [master/ed8014902436:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T22:58:42,554 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T22:58:42,577 INFO [master/ed8014902436:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T22:58:42,578 INFO [master/ed8014902436:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733266782578 2024-12-03T22:58:42,578 INFO [master/ed8014902436:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733266842578 2024-12-03T22:58:42,578 INFO [master/ed8014902436:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-12-03T22:58:42,580 INFO [master/ed8014902436:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ed8014902436,44997,1733266688559-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:42,581 INFO [master/ed8014902436:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ed8014902436,44997,1733266688559-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:42,581 INFO [master/ed8014902436:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ed8014902436,44997,1733266688559-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:42,582 INFO [master/ed8014902436:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ed8014902436:44997, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:42,583 INFO [master/ed8014902436:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:42,584 INFO [master/ed8014902436:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:42,591 DEBUG [master/ed8014902436:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T22:58:42,609 INFO [master/ed8014902436:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.129sec 2024-12-03T22:58:42,610 INFO [master/ed8014902436:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T22:58:42,611 INFO [master/ed8014902436:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T22:58:42,612 INFO [master/ed8014902436:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T22:58:42,613 INFO [master/ed8014902436:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T22:58:42,613 INFO [master/ed8014902436:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T22:58:42,614 INFO [master/ed8014902436:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ed8014902436,44997,1733266688559-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T22:58:42,614 INFO [master/ed8014902436:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ed8014902436,44997,1733266688559-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T22:58:42,623 DEBUG [master/ed8014902436:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T22:58:42,624 INFO [master/ed8014902436:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T22:58:42,624 INFO [master/ed8014902436:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ed8014902436,44997,1733266688559-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T22:58:42,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77804624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T22:58:42,664 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T22:58:42,664 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T22:58:42,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ed8014902436,44997,-1 for getting cluster id 2024-12-03T22:58:42,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T22:58:42,675 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c57f45c-be26-4ca2-b237-0e73ddd235fe' 2024-12-03T22:58:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T22:58:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c57f45c-be26-4ca2-b237-0e73ddd235fe" 2024-12-03T22:58:42,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b0ddee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T22:58:42,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ed8014902436,44997,-1] 2024-12-03T22:58:42,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T22:58:42,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T22:58:42,685 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T22:58:42,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fad6539, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T22:58:42,688 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T22:58:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ed8014902436,36731,1733266689308, seqNum=-1] 2024-12-03T22:58:42,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T22:58:42,697 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T22:58:42,716 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ed8014902436,44997,1733266688559 2024-12-03T22:58:42,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68214a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T22:58:42,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ed8014902436,44997,-1 for getting cluster id 2024-12-03T22:58:42,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T22:58:42,720 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c57f45c-be26-4ca2-b237-0e73ddd235fe' 2024-12-03T22:58:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T22:58:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c57f45c-be26-4ca2-b237-0e73ddd235fe" 2024-12-03T22:58:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58c4d5a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T22:58:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ed8014902436,44997,-1] 2024-12-03T22:58:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T22:58:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T22:58:42,723 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T22:58:42,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@666749f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T22:58:42,736 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=1234, ProcessCount=11, AvailableMemoryMB=5761 2024-12-03T22:58:42,742 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T22:58:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ed8014902436,44997,1733266688559 2024-12-03T22:58:42,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5e14daea 2024-12-03T22:58:42,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T22:58:42,752 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T22:58:42,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T22:58:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T22:58:42,767 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T22:58:42,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-03T22:58:42,769 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:42,792 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T22:58:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T22:58:42,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741835_1011 (size=428) 2024-12-03T22:58:43,214 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bbb748d5bd9330e44a775a0d924c9164, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58 2024-12-03T22:58:43,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741836_1012 (size=79) 2024-12-03T22:58:43,628 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:58:43,628 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing bbb748d5bd9330e44a775a0d924c9164, disabling compactions & flushes 2024-12-03T22:58:43,628 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. 2024-12-03T22:58:43,628 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. 2024-12-03T22:58:43,628 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. after waiting 0 ms 2024-12-03T22:58:43,628 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. 2024-12-03T22:58:43,628 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. 2024-12-03T22:58:43,628 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for bbb748d5bd9330e44a775a0d924c9164: Waiting for close lock at 1733266723628Disabling compacts and flushes for region at 1733266723628Disabling writes for close at 1733266723628Writing region close event to WAL at 1733266723628Closed at 1733266723628 2024-12-03T22:58:43,631 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T22:58:43,635 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733266723631"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733266723631"}]},"ts":"1733266723631"} 2024-12-03T22:58:43,641 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T22:58:43,643 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T22:58:43,645 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266723643"}]},"ts":"1733266723643"} 2024-12-03T22:58:43,650 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-03T22:58:43,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=bbb748d5bd9330e44a775a0d924c9164, ASSIGN}] 2024-12-03T22:58:43,654 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=bbb748d5bd9330e44a775a0d924c9164, ASSIGN 2024-12-03T22:58:43,656 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=bbb748d5bd9330e44a775a0d924c9164, ASSIGN; state=OFFLINE, location=ed8014902436,36731,1733266689308; forceNewPlan=false, retain=false 2024-12-03T22:58:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T22:58:43,809 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bbb748d5bd9330e44a775a0d924c9164, regionState=OPENING, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:58:43,816 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=bbb748d5bd9330e44a775a0d924c9164, ASSIGN because future has completed 2024-12-03T22:58:43,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbb748d5bd9330e44a775a0d924c9164, server=ed8014902436,36731,1733266689308}] 2024-12-03T22:58:43,980 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. 2024-12-03T22:58:43,981 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => bbb748d5bd9330e44a775a0d924c9164, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164.', STARTKEY => '', ENDKEY => ''} 2024-12-03T22:58:43,981 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync bbb748d5bd9330e44a775a0d924c9164 2024-12-03T22:58:43,981 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:58:43,982 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for bbb748d5bd9330e44a775a0d924c9164 2024-12-03T22:58:43,982 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for bbb748d5bd9330e44a775a0d924c9164 2024-12-03T22:58:43,984 INFO [StoreOpener-bbb748d5bd9330e44a775a0d924c9164-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region bbb748d5bd9330e44a775a0d924c9164 2024-12-03T22:58:43,987 INFO [StoreOpener-bbb748d5bd9330e44a775a0d924c9164-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb748d5bd9330e44a775a0d924c9164 columnFamilyName testFamily 2024-12-03T22:58:43,987 DEBUG [StoreOpener-bbb748d5bd9330e44a775a0d924c9164-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:43,988 INFO [StoreOpener-bbb748d5bd9330e44a775a0d924c9164-1 {}] regionserver.HStore(327): Store=bbb748d5bd9330e44a775a0d924c9164/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T22:58:43,989 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for bbb748d5bd9330e44a775a0d924c9164 2024-12-03T22:58:43,990 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bbb748d5bd9330e44a775a0d924c9164 2024-12-03T22:58:43,991 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bbb748d5bd9330e44a775a0d924c9164 2024-12-03T22:58:43,992 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for bbb748d5bd9330e44a775a0d924c9164 2024-12-03T22:58:43,992 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for bbb748d5bd9330e44a775a0d924c9164 2024-12-03T22:58:43,995 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for bbb748d5bd9330e44a775a0d924c9164 2024-12-03T22:58:43,998 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bbb748d5bd9330e44a775a0d924c9164/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T22:58:43,999 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened bbb748d5bd9330e44a775a0d924c9164; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63447436, jitterRate=-0.054559528827667236}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T22:58:43,999 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bbb748d5bd9330e44a775a0d924c9164 2024-12-03T22:58:44,000 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for bbb748d5bd9330e44a775a0d924c9164: Running coprocessor pre-open hook at 1733266723982Writing region info on filesystem at 1733266723982Initializing all the Stores at 1733266723984 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266723984Cleaning up temporary data from old regions at 1733266723992 (+8 ms)Running coprocessor post-open hooks at 1733266723999 (+7 ms)Region opened successfully at 1733266724000 (+1 ms) 2024-12-03T22:58:44,002 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164., pid=6, masterSystemTime=1733266723972 2024-12-03T22:58:44,005 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. 2024-12-03T22:58:44,005 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. 2024-12-03T22:58:44,007 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bbb748d5bd9330e44a775a0d924c9164, regionState=OPEN, openSeqNum=2, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:58:44,010 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbb748d5bd9330e44a775a0d924c9164, server=ed8014902436,36731,1733266689308 because future has completed 2024-12-03T22:58:44,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T22:58:44,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure bbb748d5bd9330e44a775a0d924c9164, server=ed8014902436,36731,1733266689308 in 195 msec 2024-12-03T22:58:44,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T22:58:44,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=bbb748d5bd9330e44a775a0d924c9164, ASSIGN in 364 msec 2024-12-03T22:58:44,021 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T22:58:44,021 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266724021"}]},"ts":"1733266724021"} 2024-12-03T22:58:44,024 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-03T22:58:44,025 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T22:58:44,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2650 sec 2024-12-03T22:58:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T22:58:45,837 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-03T22:58:45,837 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-03T22:58:45,839 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:58:45,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-03T22:58:45,848 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:58:45,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-03T22:58:45,850 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T22:58:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ed8014902436,36731,1733266689308, seqNum=-1] 2024-12-03T22:58:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T22:58:45,855 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T22:58:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164., hostname=ed8014902436,36731,1733266689308, seqNum=2] 2024-12-03T22:58:45,874 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T22:58:45,878 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-03T22:58:45,882 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-03T22:58:45,887 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-03T22:58:45,888 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-03T22:58:45,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=699721224162377736 2024-12-03T22:58:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34294 deadline: 1733266728888, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T22:58:45,912 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-03T22:58:45,913 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-03T22:58:45,914 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T22:58:45,915 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T22:58:45,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=699721224162377738 2024-12-03T22:58:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34294 deadline: 1733266728915, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T22:58:45,919 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T22:58:45,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=699721224162377739 2024-12-03T22:58:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34294 deadline: 1733266728919, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T22:58:45,923 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T22:58:45,923 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=699721224162377740 2024-12-03T22:58:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34294 deadline: 1733266728922, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T22:58:45,926 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T22:58:45,938 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1119854896_22 at /127.0.0.1:57914 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/ed8014902436:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1167 (was 1234), ProcessCount=11 (was 11), AvailableMemoryMB=5713 (was 5761) 2024-12-03T22:58:45,945 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=1167, ProcessCount=11, AvailableMemoryMB=5711 2024-12-03T22:58:45,948 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T22:58:45,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-03T22:58:45,952 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T22:58:45,952 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:45,952 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-03T22:58:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T22:58:45,954 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T22:58:45,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741837_1013 (size=414) 2024-12-03T22:58:46,369 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 897bed5ed2ecae70c69cd2703f2323f9, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58 2024-12-03T22:58:46,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741838_1014 (size=65) 2024-12-03T22:58:46,781 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:58:46,782 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 897bed5ed2ecae70c69cd2703f2323f9, disabling compactions & flushes 2024-12-03T22:58:46,782 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. 2024-12-03T22:58:46,782 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. 2024-12-03T22:58:46,782 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. after waiting 0 ms 2024-12-03T22:58:46,782 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. 2024-12-03T22:58:46,782 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. 2024-12-03T22:58:46,783 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 897bed5ed2ecae70c69cd2703f2323f9: Waiting for close lock at 1733266726782Disabling compacts and flushes for region at 1733266726782Disabling writes for close at 1733266726782Writing region close event to WAL at 1733266726782Closed at 1733266726782 2024-12-03T22:58:46,786 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T22:58:46,787 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733266726786"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733266726786"}]},"ts":"1733266726786"} 2024-12-03T22:58:46,790 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T22:58:46,791 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T22:58:46,791 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266726791"}]},"ts":"1733266726791"} 2024-12-03T22:58:46,794 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-03T22:58:46,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=897bed5ed2ecae70c69cd2703f2323f9, ASSIGN}] 2024-12-03T22:58:46,797 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=897bed5ed2ecae70c69cd2703f2323f9, ASSIGN 2024-12-03T22:58:46,798 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=897bed5ed2ecae70c69cd2703f2323f9, ASSIGN; state=OFFLINE, location=ed8014902436,36731,1733266689308; forceNewPlan=false, retain=false 2024-12-03T22:58:46,950 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=897bed5ed2ecae70c69cd2703f2323f9, regionState=OPENING, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:58:46,958 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=897bed5ed2ecae70c69cd2703f2323f9, ASSIGN because future has completed 2024-12-03T22:58:46,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 897bed5ed2ecae70c69cd2703f2323f9, server=ed8014902436,36731,1733266689308}] 2024-12-03T22:58:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T22:58:47,125 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. 2024-12-03T22:58:47,126 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 897bed5ed2ecae70c69cd2703f2323f9, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9.', STARTKEY => '', ENDKEY => ''} 2024-12-03T22:58:47,126 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 897bed5ed2ecae70c69cd2703f2323f9 2024-12-03T22:58:47,126 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:58:47,127 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 897bed5ed2ecae70c69cd2703f2323f9 2024-12-03T22:58:47,127 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 897bed5ed2ecae70c69cd2703f2323f9 2024-12-03T22:58:47,129 INFO [StoreOpener-897bed5ed2ecae70c69cd2703f2323f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 897bed5ed2ecae70c69cd2703f2323f9 2024-12-03T22:58:47,131 INFO [StoreOpener-897bed5ed2ecae70c69cd2703f2323f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 897bed5ed2ecae70c69cd2703f2323f9 columnFamilyName testFamily 2024-12-03T22:58:47,131 DEBUG [StoreOpener-897bed5ed2ecae70c69cd2703f2323f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:47,132 INFO [StoreOpener-897bed5ed2ecae70c69cd2703f2323f9-1 {}] regionserver.HStore(327): Store=897bed5ed2ecae70c69cd2703f2323f9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T22:58:47,132 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 897bed5ed2ecae70c69cd2703f2323f9 2024-12-03T22:58:47,133 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScanner/897bed5ed2ecae70c69cd2703f2323f9 2024-12-03T22:58:47,134 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScanner/897bed5ed2ecae70c69cd2703f2323f9 2024-12-03T22:58:47,135 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 897bed5ed2ecae70c69cd2703f2323f9 2024-12-03T22:58:47,135 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 897bed5ed2ecae70c69cd2703f2323f9 2024-12-03T22:58:47,137 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 897bed5ed2ecae70c69cd2703f2323f9 2024-12-03T22:58:47,141 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScanner/897bed5ed2ecae70c69cd2703f2323f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T22:58:47,142 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 897bed5ed2ecae70c69cd2703f2323f9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58901952, jitterRate=-0.12229251861572266}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T22:58:47,142 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 897bed5ed2ecae70c69cd2703f2323f9 2024-12-03T22:58:47,143 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 897bed5ed2ecae70c69cd2703f2323f9: Running coprocessor pre-open hook at 1733266727127Writing region info on filesystem at 1733266727127Initializing all the Stores at 1733266727129 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266727129Cleaning up temporary data from old regions at 1733266727135 (+6 ms)Running coprocessor post-open hooks at 1733266727142 (+7 ms)Region opened successfully at 1733266727143 (+1 ms) 2024-12-03T22:58:47,144 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9., pid=9, masterSystemTime=1733266727113 2024-12-03T22:58:47,148 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. 2024-12-03T22:58:47,148 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. 2024-12-03T22:58:47,149 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=897bed5ed2ecae70c69cd2703f2323f9, regionState=OPEN, openSeqNum=2, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:58:47,152 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 897bed5ed2ecae70c69cd2703f2323f9, server=ed8014902436,36731,1733266689308 because future has completed 2024-12-03T22:58:47,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-03T22:58:47,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 897bed5ed2ecae70c69cd2703f2323f9, server=ed8014902436,36731,1733266689308 in 195 msec 2024-12-03T22:58:47,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T22:58:47,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=897bed5ed2ecae70c69cd2703f2323f9, ASSIGN in 362 msec 2024-12-03T22:58:47,162 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T22:58:47,163 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266727162"}]},"ts":"1733266727162"} 2024-12-03T22:58:47,166 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-03T22:58:47,167 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T22:58:47,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2190 sec 2024-12-03T22:58:47,917 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T22:58:47,943 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-03T22:58:47,944 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T22:58:47,945 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-03T22:58:48,930 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T22:58:48,930 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T22:58:48,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T22:58:48,932 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T22:58:48,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T22:58:48,933 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T22:58:48,934 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-03T22:58:48,934 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-03T22:58:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T22:58:48,972 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-03T22:58:48,973 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-03T22:58:48,973 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:58:48,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-03T22:58:48,979 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:58:48,979 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-03T22:58:48,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9., hostname=ed8014902436,36731,1733266689308, seqNum=2] 2024-12-03T22:58:48,996 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T22:58:48,996 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-03T22:58:49,000 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T22:58:49,002 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-03T22:58:55,022 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9., hostname=ed8014902436,36731,1733266689308, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9., hostname=ed8014902436,36731,1733266689308, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ed8014902436:36731 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6016ms, rpcTimeout=5999ms 2024-12-03T22:58:55,023 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9., hostname=ed8014902436,36731,1733266689308, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6016ms, rpcTimeout=5999ms 2024-12-03T22:58:55,023 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9., hostname=ed8014902436,36731,1733266689308, seqNum=2 from cache 2024-12-03T22:58:55,034 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=988 (was 1167), ProcessCount=11 (was 11), AvailableMemoryMB=5700 (was 5711) 2024-12-03T22:58:55,041 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=988, ProcessCount=11, AvailableMemoryMB=5699 2024-12-03T22:58:55,044 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T22:58:55,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-03T22:58:55,048 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T22:58:55,048 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:55,048 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-03T22:58:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T22:58:55,050 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T22:58:55,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741839_1015 (size=412) 2024-12-03T22:58:55,467 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 78246d6932357f4d3bdead6e1777c4c5, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58 2024-12-03T22:58:55,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741840_1016 (size=63) 2024-12-03T22:58:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-12-03T22:58:55,882 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:58:55,882 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 78246d6932357f4d3bdead6e1777c4c5, disabling compactions & flushes 2024-12-03T22:58:55,882 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. 2024-12-03T22:58:55,883 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. 2024-12-03T22:58:55,883 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. after waiting 0 ms 2024-12-03T22:58:55,883 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. 2024-12-03T22:58:55,883 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. 2024-12-03T22:58:55,884 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 78246d6932357f4d3bdead6e1777c4c5: Waiting for close lock at 1733266735882Disabling compacts and flushes for region at 1733266735882Disabling writes for close at 1733266735883 (+1 ms)Writing region close event to WAL at 1733266735883Closed at 1733266735883 2024-12-03T22:58:55,886 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T22:58:55,887 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733266735886"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733266735886"}]},"ts":"1733266735886"} 2024-12-03T22:58:55,890 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T22:58:55,891 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T22:58:55,891 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266735891"}]},"ts":"1733266735891"} 2024-12-03T22:58:55,893 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-03T22:58:55,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=78246d6932357f4d3bdead6e1777c4c5, ASSIGN}] 2024-12-03T22:58:55,895 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=78246d6932357f4d3bdead6e1777c4c5, ASSIGN 2024-12-03T22:58:55,896 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=78246d6932357f4d3bdead6e1777c4c5, ASSIGN; state=OFFLINE, location=ed8014902436,36731,1733266689308; forceNewPlan=false, retain=false 2024-12-03T22:58:56,048 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=78246d6932357f4d3bdead6e1777c4c5, regionState=OPENING, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:58:56,058 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=78246d6932357f4d3bdead6e1777c4c5, ASSIGN because future has completed 2024-12-03T22:58:56,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78246d6932357f4d3bdead6e1777c4c5, server=ed8014902436,36731,1733266689308}] 2024-12-03T22:58:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T22:58:56,226 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. 2024-12-03T22:58:56,226 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 78246d6932357f4d3bdead6e1777c4c5, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5.', STARTKEY => '', ENDKEY => ''} 2024-12-03T22:58:56,226 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:58:56,227 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:58:56,227 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:58:56,227 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:58:56,229 INFO [StoreOpener-78246d6932357f4d3bdead6e1777c4c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:58:56,232 INFO [StoreOpener-78246d6932357f4d3bdead6e1777c4c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78246d6932357f4d3bdead6e1777c4c5 columnFamilyName testFamily 2024-12-03T22:58:56,232 DEBUG [StoreOpener-78246d6932357f4d3bdead6e1777c4c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:58:56,233 INFO [StoreOpener-78246d6932357f4d3bdead6e1777c4c5-1 {}] regionserver.HStore(327): Store=78246d6932357f4d3bdead6e1777c4c5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T22:58:56,233 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:58:56,235 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNextAsync/78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:58:56,236 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNextAsync/78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:58:56,236 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:58:56,236 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:58:56,240 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:58:56,244 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNextAsync/78246d6932357f4d3bdead6e1777c4c5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T22:58:56,246 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 78246d6932357f4d3bdead6e1777c4c5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61946861, jitterRate=-0.07691983878612518}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T22:58:56,246 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:58:56,247 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 78246d6932357f4d3bdead6e1777c4c5: Running coprocessor pre-open hook at 1733266736227Writing region info on filesystem at 1733266736227Initializing all the Stores at 1733266736229 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266736229Cleaning up temporary data from old regions at 1733266736237 (+8 ms)Running coprocessor post-open hooks at 1733266736246 (+9 ms)Region opened successfully at 1733266736247 (+1 ms) 2024-12-03T22:58:56,248 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5., pid=12, masterSystemTime=1733266736214 2024-12-03T22:58:56,251 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. 2024-12-03T22:58:56,251 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. 2024-12-03T22:58:56,252 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=78246d6932357f4d3bdead6e1777c4c5, regionState=OPEN, openSeqNum=2, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:58:56,256 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78246d6932357f4d3bdead6e1777c4c5, server=ed8014902436,36731,1733266689308 because future has completed 2024-12-03T22:58:56,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T22:58:56,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 78246d6932357f4d3bdead6e1777c4c5, server=ed8014902436,36731,1733266689308 in 198 msec 2024-12-03T22:58:56,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-03T22:58:56,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=78246d6932357f4d3bdead6e1777c4c5, ASSIGN in 367 msec 2024-12-03T22:58:56,266 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T22:58:56,266 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266736266"}]},"ts":"1733266736266"} 2024-12-03T22:58:56,269 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-03T22:58:56,271 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T22:58:56,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2270 sec 2024-12-03T22:58:58,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T22:58:58,073 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-03T22:58:58,073 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-03T22:58:58,074 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:58:58,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-03T22:58:58,080 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:58:58,080 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-03T22:58:58,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5., hostname=ed8014902436,36731,1733266689308, seqNum=2] 2024-12-03T22:58:58,098 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T22:58:58,098 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-03T22:58:58,102 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T22:58:58,104 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T22:58:58,106 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T22:58:58,106 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-03T22:58:58,930 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-03T22:58:58,930 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T22:58:59,946 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-03T22:59:01,760 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T22:59:04,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36731 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '699721224162377749' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T22:59:04,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36731 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:34294 deadline: 1733266750112, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '699721224162377749' does not exist 2024-12-03T22:59:04,115 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to ed8014902436,36731,1733266689308 for closing scanner id = 699721224162377749 for 78246d6932357f4d3bdead6e1777c4c5 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '699721224162377749' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T22:59:04,117 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=836 (was 988), ProcessCount=11 (was 11), AvailableMemoryMB=5690 (was 5699) 2024-12-03T22:59:04,124 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=836, ProcessCount=11, AvailableMemoryMB=5690 2024-12-03T22:59:04,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T22:59:04,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-03T22:59:04,130 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T22:59:04,131 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:59:04,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-03T22:59:04,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T22:59:04,132 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T22:59:04,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741841_1017 (size=423) 2024-12-03T22:59:04,547 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3cd9eb5ad6480573474415850e7c0740, NAME => 'testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58 2024-12-03T22:59:04,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741842_1018 (size=74) 2024-12-03T22:59:04,559 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:59:04,559 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 3cd9eb5ad6480573474415850e7c0740, disabling compactions & flushes 2024-12-03T22:59:04,559 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. 2024-12-03T22:59:04,559 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. 2024-12-03T22:59:04,559 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. after waiting 0 ms 2024-12-03T22:59:04,559 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. 2024-12-03T22:59:04,559 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. 2024-12-03T22:59:04,560 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3cd9eb5ad6480573474415850e7c0740: Waiting for close lock at 1733266744559Disabling compacts and flushes for region at 1733266744559Disabling writes for close at 1733266744559Writing region close event to WAL at 1733266744559Closed at 1733266744559 2024-12-03T22:59:04,561 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T22:59:04,562 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733266744561"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733266744561"}]},"ts":"1733266744561"} 2024-12-03T22:59:04,565 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T22:59:04,566 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T22:59:04,566 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266744566"}]},"ts":"1733266744566"} 2024-12-03T22:59:04,569 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-03T22:59:04,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=3cd9eb5ad6480573474415850e7c0740, ASSIGN}] 2024-12-03T22:59:04,571 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=3cd9eb5ad6480573474415850e7c0740, ASSIGN 2024-12-03T22:59:04,572 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=3cd9eb5ad6480573474415850e7c0740, ASSIGN; state=OFFLINE, location=ed8014902436,36731,1733266689308; forceNewPlan=false, retain=false 2024-12-03T22:59:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-12-03T22:59:04,723 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=3cd9eb5ad6480573474415850e7c0740, regionState=OPENING, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:59:04,730 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=3cd9eb5ad6480573474415850e7c0740, ASSIGN because future has completed 2024-12-03T22:59:04,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3cd9eb5ad6480573474415850e7c0740, server=ed8014902436,36731,1733266689308}] 2024-12-03T22:59:04,900 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. 2024-12-03T22:59:04,901 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 3cd9eb5ad6480573474415850e7c0740, NAME => 'testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740.', STARTKEY => '', ENDKEY => ''} 2024-12-03T22:59:04,901 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 3cd9eb5ad6480573474415850e7c0740 2024-12-03T22:59:04,901 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:59:04,902 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 3cd9eb5ad6480573474415850e7c0740 2024-12-03T22:59:04,902 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 3cd9eb5ad6480573474415850e7c0740 2024-12-03T22:59:04,903 INFO [StoreOpener-3cd9eb5ad6480573474415850e7c0740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3cd9eb5ad6480573474415850e7c0740 2024-12-03T22:59:04,906 INFO [StoreOpener-3cd9eb5ad6480573474415850e7c0740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3cd9eb5ad6480573474415850e7c0740 columnFamilyName testFamily 2024-12-03T22:59:04,907 DEBUG [StoreOpener-3cd9eb5ad6480573474415850e7c0740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:59:04,907 INFO [StoreOpener-3cd9eb5ad6480573474415850e7c0740-1 {}] regionserver.HStore(327): Store=3cd9eb5ad6480573474415850e7c0740/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T22:59:04,907 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 3cd9eb5ad6480573474415850e7c0740 2024-12-03T22:59:04,908 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextException/3cd9eb5ad6480573474415850e7c0740 2024-12-03T22:59:04,909 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextException/3cd9eb5ad6480573474415850e7c0740 2024-12-03T22:59:04,910 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 3cd9eb5ad6480573474415850e7c0740 2024-12-03T22:59:04,910 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 3cd9eb5ad6480573474415850e7c0740 2024-12-03T22:59:04,912 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 3cd9eb5ad6480573474415850e7c0740 2024-12-03T22:59:04,916 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextException/3cd9eb5ad6480573474415850e7c0740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T22:59:04,917 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 3cd9eb5ad6480573474415850e7c0740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74265921, jitterRate=0.10664846003055573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T22:59:04,917 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3cd9eb5ad6480573474415850e7c0740 2024-12-03T22:59:04,918 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 3cd9eb5ad6480573474415850e7c0740: Running coprocessor pre-open hook at 1733266744902Writing region info on filesystem at 1733266744902Initializing all the Stores at 1733266744903 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266744903Cleaning up temporary data from old regions at 1733266744910 (+7 ms)Running coprocessor post-open hooks at 1733266744917 (+7 ms)Region opened successfully at 1733266744918 (+1 ms) 2024-12-03T22:59:04,919 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740., pid=15, masterSystemTime=1733266744888 2024-12-03T22:59:04,923 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. 2024-12-03T22:59:04,923 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. 2024-12-03T22:59:04,924 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=3cd9eb5ad6480573474415850e7c0740, regionState=OPEN, openSeqNum=2, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:59:04,926 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3cd9eb5ad6480573474415850e7c0740, server=ed8014902436,36731,1733266689308 because future has completed 2024-12-03T22:59:04,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-03T22:59:04,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 3cd9eb5ad6480573474415850e7c0740, server=ed8014902436,36731,1733266689308 in 196 msec 2024-12-03T22:59:04,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T22:59:04,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=3cd9eb5ad6480573474415850e7c0740, ASSIGN in 363 msec 2024-12-03T22:59:04,936 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T22:59:04,936 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266744936"}]},"ts":"1733266744936"} 2024-12-03T22:59:04,939 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-03T22:59:04,940 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T22:59:04,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 813 msec 2024-12-03T22:59:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T22:59:05,153 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-03T22:59:05,154 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-03T22:59:05,154 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:59:05,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-03T22:59:05,162 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:59:05,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-03T22:59:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740., hostname=ed8014902436,36731,1733266689308, seqNum=2] 2024-12-03T22:59:05,178 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T22:59:05,179 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-03T22:59:05,183 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-03T22:59:05,186 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-03T22:59:05,187 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-03T22:59:05,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=699721224162377753 2024-12-03T22:59:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34294 deadline: 1733266748187, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T22:59:05,193 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-03T22:59:05,195 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-03T22:59:05,196 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T22:59:05,198 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T22:59:05,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=699721224162377755 2024-12-03T22:59:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34294 deadline: 1733266748197, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T22:59:05,203 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T22:59:05,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=699721224162377756 2024-12-03T22:59:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34294 deadline: 1733266748202, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T22:59:05,206 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T22:59:05,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=699721224162377757 2024-12-03T22:59:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34294 deadline: 1733266748205, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T22:59:05,209 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T22:59:05,217 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=228 (was 225) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1119854896_22 at /127.0.0.1:43454 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=836 (was 836), ProcessCount=11 (was 11), AvailableMemoryMB=5686 (was 5690) 2024-12-03T22:59:05,225 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=228, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=836, ProcessCount=11, AvailableMemoryMB=5686 2024-12-03T22:59:05,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T22:59:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-03T22:59:05,231 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T22:59:05,231 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:59:05,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-03T22:59:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T22:59:05,233 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T22:59:05,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741843_1019 (size=418) 2024-12-03T22:59:05,651 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a4bff265cf897b5304f15f9f43579d6c, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58 2024-12-03T22:59:05,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741844_1020 (size=69) 2024-12-03T22:59:05,948 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-03T22:59:06,060 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:59:06,060 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing a4bff265cf897b5304f15f9f43579d6c, disabling compactions & flushes 2024-12-03T22:59:06,060 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. 2024-12-03T22:59:06,060 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. 2024-12-03T22:59:06,060 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. after waiting 0 ms 2024-12-03T22:59:06,061 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. 2024-12-03T22:59:06,061 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. 2024-12-03T22:59:06,061 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for a4bff265cf897b5304f15f9f43579d6c: Waiting for close lock at 1733266746060Disabling compacts and flushes for region at 1733266746060Disabling writes for close at 1733266746060Writing region close event to WAL at 1733266746061 (+1 ms)Closed at 1733266746061 2024-12-03T22:59:06,064 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T22:59:06,065 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733266746064"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733266746064"}]},"ts":"1733266746064"} 2024-12-03T22:59:06,070 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T22:59:06,071 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T22:59:06,072 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266746071"}]},"ts":"1733266746071"} 2024-12-03T22:59:06,074 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-03T22:59:06,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=a4bff265cf897b5304f15f9f43579d6c, ASSIGN}] 2024-12-03T22:59:06,076 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=a4bff265cf897b5304f15f9f43579d6c, ASSIGN 2024-12-03T22:59:06,078 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=a4bff265cf897b5304f15f9f43579d6c, ASSIGN; state=OFFLINE, location=ed8014902436,36731,1733266689308; forceNewPlan=false, retain=false 2024-12-03T22:59:06,229 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=a4bff265cf897b5304f15f9f43579d6c, regionState=OPENING, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:59:06,234 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=a4bff265cf897b5304f15f9f43579d6c, ASSIGN because future has completed 2024-12-03T22:59:06,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4bff265cf897b5304f15f9f43579d6c, server=ed8014902436,36731,1733266689308}] 2024-12-03T22:59:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T22:59:06,402 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. 2024-12-03T22:59:06,402 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => a4bff265cf897b5304f15f9f43579d6c, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c.', STARTKEY => '', ENDKEY => ''} 2024-12-03T22:59:06,402 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner a4bff265cf897b5304f15f9f43579d6c 2024-12-03T22:59:06,402 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:59:06,403 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for a4bff265cf897b5304f15f9f43579d6c 2024-12-03T22:59:06,403 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for a4bff265cf897b5304f15f9f43579d6c 2024-12-03T22:59:06,405 INFO [StoreOpener-a4bff265cf897b5304f15f9f43579d6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a4bff265cf897b5304f15f9f43579d6c 2024-12-03T22:59:06,407 INFO [StoreOpener-a4bff265cf897b5304f15f9f43579d6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4bff265cf897b5304f15f9f43579d6c columnFamilyName testFamily 2024-12-03T22:59:06,407 DEBUG [StoreOpener-a4bff265cf897b5304f15f9f43579d6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:59:06,408 INFO [StoreOpener-a4bff265cf897b5304f15f9f43579d6c-1 {}] regionserver.HStore(327): Store=a4bff265cf897b5304f15f9f43579d6c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T22:59:06,408 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for a4bff265cf897b5304f15f9f43579d6c 2024-12-03T22:59:06,409 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScanner/a4bff265cf897b5304f15f9f43579d6c 2024-12-03T22:59:06,409 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScanner/a4bff265cf897b5304f15f9f43579d6c 2024-12-03T22:59:06,410 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for a4bff265cf897b5304f15f9f43579d6c 2024-12-03T22:59:06,410 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for a4bff265cf897b5304f15f9f43579d6c 2024-12-03T22:59:06,412 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for a4bff265cf897b5304f15f9f43579d6c 2024-12-03T22:59:06,415 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScanner/a4bff265cf897b5304f15f9f43579d6c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T22:59:06,416 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened a4bff265cf897b5304f15f9f43579d6c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60362558, jitterRate=-0.1005277931690216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T22:59:06,416 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4bff265cf897b5304f15f9f43579d6c 2024-12-03T22:59:06,416 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for a4bff265cf897b5304f15f9f43579d6c: Running coprocessor pre-open hook at 1733266746403Writing region info on filesystem at 1733266746403Initializing all the Stores at 1733266746404 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266746405 (+1 ms)Cleaning up temporary data from old regions at 1733266746410 (+5 ms)Running coprocessor post-open hooks at 1733266746416 (+6 ms)Region opened successfully at 1733266746416 2024-12-03T22:59:06,417 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c., pid=18, masterSystemTime=1733266746391 2024-12-03T22:59:06,420 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. 2024-12-03T22:59:06,420 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. 2024-12-03T22:59:06,421 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=a4bff265cf897b5304f15f9f43579d6c, regionState=OPEN, openSeqNum=2, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:59:06,424 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4bff265cf897b5304f15f9f43579d6c, server=ed8014902436,36731,1733266689308 because future has completed 2024-12-03T22:59:06,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T22:59:06,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure a4bff265cf897b5304f15f9f43579d6c, server=ed8014902436,36731,1733266689308 in 191 msec 2024-12-03T22:59:06,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-03T22:59:06,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=a4bff265cf897b5304f15f9f43579d6c, ASSIGN in 355 msec 2024-12-03T22:59:06,435 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T22:59:06,435 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266746435"}]},"ts":"1733266746435"} 2024-12-03T22:59:06,438 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-03T22:59:06,439 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T22:59:06,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2110 sec 2024-12-03T22:59:07,539 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T22:59:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T22:59:08,272 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-03T22:59:08,272 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-03T22:59:08,272 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:59:08,276 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-03T22:59:08,276 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:59:08,276 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-03T22:59:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c., hostname=ed8014902436,36731,1733266689308, seqNum=2] 2024-12-03T22:59:08,291 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T22:59:08,291 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-03T22:59:08,295 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T22:59:08,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T22:59:08,930 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-03T22:59:08,931 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-03T22:59:08,933 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-03T22:59:08,933 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-03T22:59:09,299 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c., hostname=ed8014902436,36731,1733266689308, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c., hostname=ed8014902436,36731,1733266689308, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ed8014902436:36731 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-12-03T22:59:09,300 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c., hostname=ed8014902436,36731,1733266689308, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-12-03T22:59:09,300 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c., hostname=ed8014902436,36731,1733266689308, seqNum=2 from cache 2024-12-03T22:59:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-12-03T22:59:10,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c., hostname=ed8014902436,36731,1733266689308, seqNum=2] 2024-12-03T22:59:10,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T22:59:10,405 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T22:59:11,299 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c., hostname=ed8014902436,36731,1733266689308, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c., hostname=ed8014902436,36731,1733266689308, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ed8014902436:36731 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=985ms, rpcTimeout=981ms 2024-12-03T22:59:11,300 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c., hostname=ed8014902436,36731,1733266689308, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=985ms, rpcTimeout=981ms 2024-12-03T22:59:11,300 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c., hostname=ed8014902436,36731,1733266689308, seqNum=2 from cache 2024-12-03T22:59:11,316 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 228), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=769 (was 836), ProcessCount=11 (was 11), AvailableMemoryMB=5673 (was 5686) 2024-12-03T22:59:11,324 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=769, ProcessCount=11, AvailableMemoryMB=5673 2024-12-03T22:59:11,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T22:59:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-03T22:59:11,329 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T22:59:11,329 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:59:11,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-03T22:59:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T22:59:11,331 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T22:59:11,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741845_1021 (size=416) 2024-12-03T22:59:11,742 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 46d22f3dadfd475287722d11d378de03, NAME => 'testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58 2024-12-03T22:59:11,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741846_1022 (size=67) 2024-12-03T22:59:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-12-03T22:59:11,952 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-03T22:59:12,157 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:59:12,157 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 46d22f3dadfd475287722d11d378de03, disabling compactions & flushes 2024-12-03T22:59:12,157 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. 2024-12-03T22:59:12,157 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. 2024-12-03T22:59:12,157 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. after waiting 0 ms 2024-12-03T22:59:12,158 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. 2024-12-03T22:59:12,158 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. 2024-12-03T22:59:12,158 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 46d22f3dadfd475287722d11d378de03: Waiting for close lock at 1733266752157Disabling compacts and flushes for region at 1733266752157Disabling writes for close at 1733266752157Writing region close event to WAL at 1733266752158 (+1 ms)Closed at 1733266752158 2024-12-03T22:59:12,195 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T22:59:12,195 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733266752195"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733266752195"}]},"ts":"1733266752195"} 2024-12-03T22:59:12,200 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T22:59:12,203 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T22:59:12,203 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266752203"}]},"ts":"1733266752203"} 2024-12-03T22:59:12,207 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-03T22:59:12,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=46d22f3dadfd475287722d11d378de03, ASSIGN}] 2024-12-03T22:59:12,209 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=46d22f3dadfd475287722d11d378de03, ASSIGN 2024-12-03T22:59:12,211 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=46d22f3dadfd475287722d11d378de03, ASSIGN; state=OFFLINE, location=ed8014902436,36731,1733266689308; forceNewPlan=false, retain=false 2024-12-03T22:59:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T22:59:12,362 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=46d22f3dadfd475287722d11d378de03, regionState=OPENING, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:59:12,365 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=46d22f3dadfd475287722d11d378de03, ASSIGN because future has completed 2024-12-03T22:59:12,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46d22f3dadfd475287722d11d378de03, server=ed8014902436,36731,1733266689308}] 2024-12-03T22:59:12,532 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. 2024-12-03T22:59:12,532 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 46d22f3dadfd475287722d11d378de03, NAME => 'testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03.', STARTKEY => '', ENDKEY => ''} 2024-12-03T22:59:12,533 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 46d22f3dadfd475287722d11d378de03 2024-12-03T22:59:12,533 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:59:12,533 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 46d22f3dadfd475287722d11d378de03 2024-12-03T22:59:12,533 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 46d22f3dadfd475287722d11d378de03 2024-12-03T22:59:12,535 INFO [StoreOpener-46d22f3dadfd475287722d11d378de03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 46d22f3dadfd475287722d11d378de03 2024-12-03T22:59:12,538 INFO [StoreOpener-46d22f3dadfd475287722d11d378de03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46d22f3dadfd475287722d11d378de03 columnFamilyName testFamily 2024-12-03T22:59:12,538 DEBUG [StoreOpener-46d22f3dadfd475287722d11d378de03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:59:12,538 INFO [StoreOpener-46d22f3dadfd475287722d11d378de03-1 {}] regionserver.HStore(327): Store=46d22f3dadfd475287722d11d378de03/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T22:59:12,539 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 46d22f3dadfd475287722d11d378de03 2024-12-03T22:59:12,540 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNextAsync/46d22f3dadfd475287722d11d378de03 2024-12-03T22:59:12,540 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNextAsync/46d22f3dadfd475287722d11d378de03 2024-12-03T22:59:12,541 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 46d22f3dadfd475287722d11d378de03 2024-12-03T22:59:12,541 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 46d22f3dadfd475287722d11d378de03 2024-12-03T22:59:12,543 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 46d22f3dadfd475287722d11d378de03 2024-12-03T22:59:12,547 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNextAsync/46d22f3dadfd475287722d11d378de03/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T22:59:12,548 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 46d22f3dadfd475287722d11d378de03; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73890324, jitterRate=0.10105162858963013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T22:59:12,548 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 46d22f3dadfd475287722d11d378de03 2024-12-03T22:59:12,548 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 46d22f3dadfd475287722d11d378de03: Running coprocessor pre-open hook at 1733266752533Writing region info on filesystem at 1733266752533Initializing all the Stores at 1733266752535 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266752535Cleaning up temporary data from old regions at 1733266752541 (+6 ms)Running coprocessor post-open hooks at 1733266752548 (+7 ms)Region opened successfully at 1733266752548 2024-12-03T22:59:12,550 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03., pid=21, masterSystemTime=1733266752519 2024-12-03T22:59:12,553 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. 2024-12-03T22:59:12,553 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. 2024-12-03T22:59:12,554 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=46d22f3dadfd475287722d11d378de03, regionState=OPEN, openSeqNum=2, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:59:12,557 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46d22f3dadfd475287722d11d378de03, server=ed8014902436,36731,1733266689308 because future has completed 2024-12-03T22:59:12,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-03T22:59:12,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 46d22f3dadfd475287722d11d378de03, server=ed8014902436,36731,1733266689308 in 193 msec 2024-12-03T22:59:12,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T22:59:12,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=46d22f3dadfd475287722d11d378de03, ASSIGN in 354 msec 2024-12-03T22:59:12,567 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T22:59:12,568 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266752567"}]},"ts":"1733266752567"} 2024-12-03T22:59:12,570 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-03T22:59:12,572 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T22:59:12,575 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2460 sec 2024-12-03T22:59:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T22:59:14,353 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-03T22:59:14,353 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-03T22:59:14,353 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:59:14,359 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-03T22:59:14,360 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:59:14,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-03T22:59:14,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03., hostname=ed8014902436,36731,1733266689308, seqNum=2] 2024-12-03T22:59:14,375 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T22:59:14,376 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-03T22:59:14,379 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T22:59:14,381 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T22:59:14,382 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T22:59:14,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-03T22:59:17,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36731 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '699721224162377768' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T22:59:17,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36731 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:34294 deadline: 1733266758394, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '699721224162377768' does not exist 2024-12-03T22:59:17,397 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to ed8014902436,36731,1733266689308 for closing scanner id = 699721224162377768 for 46d22f3dadfd475287722d11d378de03 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '699721224162377768' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T22:59:17,407 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=715 (was 769), ProcessCount=11 (was 11), AvailableMemoryMB=5673 (was 5673) 2024-12-03T22:59:17,414 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=715, ProcessCount=11, AvailableMemoryMB=5673 2024-12-03T22:59:17,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T22:59:17,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-03T22:59:17,428 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T22:59:17,428 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:59:17,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-03T22:59:17,429 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T22:59:17,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T22:59:17,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741847_1023 (size=411) 2024-12-03T22:59:17,845 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e048d04e396089b38cef2a0e42e3fdf4, NAME => 'testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58 2024-12-03T22:59:17,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741848_1024 (size=62) 2024-12-03T22:59:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-12-03T22:59:18,039 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T22:59:18,066 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-03T22:59:18,259 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:59:18,259 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing e048d04e396089b38cef2a0e42e3fdf4, disabling compactions & flushes 2024-12-03T22:59:18,260 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. 2024-12-03T22:59:18,260 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. 2024-12-03T22:59:18,260 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. after waiting 0 ms 2024-12-03T22:59:18,260 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. 2024-12-03T22:59:18,260 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. 2024-12-03T22:59:18,260 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for e048d04e396089b38cef2a0e42e3fdf4: Waiting for close lock at 1733266758259Disabling compacts and flushes for region at 1733266758259Disabling writes for close at 1733266758260 (+1 ms)Writing region close event to WAL at 1733266758260Closed at 1733266758260 2024-12-03T22:59:18,262 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T22:59:18,263 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733266758262"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733266758262"}]},"ts":"1733266758262"} 2024-12-03T22:59:18,267 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T22:59:18,269 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T22:59:18,270 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266758269"}]},"ts":"1733266758269"} 2024-12-03T22:59:18,274 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-03T22:59:18,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e048d04e396089b38cef2a0e42e3fdf4, ASSIGN}] 2024-12-03T22:59:18,276 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e048d04e396089b38cef2a0e42e3fdf4, ASSIGN 2024-12-03T22:59:18,277 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e048d04e396089b38cef2a0e42e3fdf4, ASSIGN; state=OFFLINE, location=ed8014902436,36731,1733266689308; forceNewPlan=false, retain=false 2024-12-03T22:59:18,428 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e048d04e396089b38cef2a0e42e3fdf4, regionState=OPENING, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:59:18,436 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e048d04e396089b38cef2a0e42e3fdf4, ASSIGN because future has completed 2024-12-03T22:59:18,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure e048d04e396089b38cef2a0e42e3fdf4, server=ed8014902436,36731,1733266689308}] 2024-12-03T22:59:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T22:59:18,603 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. 2024-12-03T22:59:18,603 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => e048d04e396089b38cef2a0e42e3fdf4, NAME => 'testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4.', STARTKEY => '', ENDKEY => ''} 2024-12-03T22:59:18,604 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:18,604 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:59:18,604 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:18,605 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:18,607 INFO [StoreOpener-e048d04e396089b38cef2a0e42e3fdf4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:18,610 INFO [StoreOpener-e048d04e396089b38cef2a0e42e3fdf4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e048d04e396089b38cef2a0e42e3fdf4 columnFamilyName testFamily 2024-12-03T22:59:18,610 DEBUG [StoreOpener-e048d04e396089b38cef2a0e42e3fdf4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:59:18,611 INFO [StoreOpener-e048d04e396089b38cef2a0e42e3fdf4-1 {}] regionserver.HStore(327): Store=e048d04e396089b38cef2a0e42e3fdf4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T22:59:18,611 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:18,612 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNext/e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:18,613 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNext/e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:18,614 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:18,614 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:18,617 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:18,621 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNext/e048d04e396089b38cef2a0e42e3fdf4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T22:59:18,623 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened e048d04e396089b38cef2a0e42e3fdf4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64628288, jitterRate=-0.036963462829589844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T22:59:18,623 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:18,624 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for e048d04e396089b38cef2a0e42e3fdf4: Running coprocessor pre-open hook at 1733266758605Writing region info on filesystem at 1733266758605Initializing all the Stores at 1733266758607 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266758607Cleaning up temporary data from old regions at 1733266758614 (+7 ms)Running coprocessor post-open hooks at 1733266758623 (+9 ms)Region opened successfully at 1733266758623 2024-12-03T22:59:18,625 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4., pid=24, masterSystemTime=1733266758591 2024-12-03T22:59:18,628 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. 2024-12-03T22:59:18,628 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. 2024-12-03T22:59:18,628 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e048d04e396089b38cef2a0e42e3fdf4, regionState=OPEN, openSeqNum=2, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:59:18,631 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure e048d04e396089b38cef2a0e42e3fdf4, server=ed8014902436,36731,1733266689308 because future has completed 2024-12-03T22:59:18,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-03T22:59:18,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure e048d04e396089b38cef2a0e42e3fdf4, server=ed8014902436,36731,1733266689308 in 196 msec 2024-12-03T22:59:18,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-03T22:59:18,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e048d04e396089b38cef2a0e42e3fdf4, ASSIGN in 360 msec 2024-12-03T22:59:18,638 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T22:59:18,638 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266758638"}]},"ts":"1733266758638"} 2024-12-03T22:59:18,641 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-03T22:59:18,642 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T22:59:18,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2260 sec 2024-12-03T22:59:18,930 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-03T22:59:18,930 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-03T22:59:18,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-03T22:59:18,932 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T22:59:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T22:59:20,455 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-03T22:59:20,456 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-03T22:59:20,456 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:59:20,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-03T22:59:20,465 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:59:20,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-03T22:59:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4., hostname=ed8014902436,36731,1733266689308, seqNum=2] 2024-12-03T22:59:20,481 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T22:59:20,481 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-03T22:59:20,485 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T22:59:20,488 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T22:59:20,490 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T22:59:20,491 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-03T22:59:23,289 INFO [master/ed8014902436:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T22:59:23,289 INFO [master/ed8014902436:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T22:59:23,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36731 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '699721224162377772' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T22:59:23,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36731 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:34294 deadline: 1733266764507, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '699721224162377772' does not exist 2024-12-03T22:59:23,508 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to ed8014902436,36731,1733266689308 for closing scanner id = 699721224162377772 for e048d04e396089b38cef2a0e42e3fdf4 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '699721224162377772' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T22:59:23,515 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=605 (was 715), ProcessCount=11 (was 11), AvailableMemoryMB=5663 (was 5673) 2024-12-03T22:59:23,523 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=605, ProcessCount=11, AvailableMemoryMB=5663 2024-12-03T22:59:23,525 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T22:59:23,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T22:59:23,528 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T22:59:23,529 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:59:23,529 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-03T22:59:23,530 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T22:59:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T22:59:23,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741849_1025 (size=419) 2024-12-03T22:59:23,946 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d9c7e273746924274193a9471bc0898e, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58 2024-12-03T22:59:23,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741850_1026 (size=70) 2024-12-03T22:59:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-12-03T22:59:24,108 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T22:59:24,138 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-03T22:59:24,360 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:59:24,360 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing d9c7e273746924274193a9471bc0898e, disabling compactions & flushes 2024-12-03T22:59:24,361 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. 2024-12-03T22:59:24,361 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. 2024-12-03T22:59:24,361 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. after waiting 0 ms 2024-12-03T22:59:24,361 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. 2024-12-03T22:59:24,361 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. 2024-12-03T22:59:24,362 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for d9c7e273746924274193a9471bc0898e: Waiting for close lock at 1733266764360Disabling compacts and flushes for region at 1733266764360Disabling writes for close at 1733266764361 (+1 ms)Writing region close event to WAL at 1733266764361Closed at 1733266764361 2024-12-03T22:59:24,364 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T22:59:24,364 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733266764364"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733266764364"}]},"ts":"1733266764364"} 2024-12-03T22:59:24,369 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T22:59:24,370 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T22:59:24,371 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266764370"}]},"ts":"1733266764370"} 2024-12-03T22:59:24,373 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-03T22:59:24,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d9c7e273746924274193a9471bc0898e, ASSIGN}] 2024-12-03T22:59:24,375 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d9c7e273746924274193a9471bc0898e, ASSIGN 2024-12-03T22:59:24,376 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d9c7e273746924274193a9471bc0898e, ASSIGN; state=OFFLINE, location=ed8014902436,36731,1733266689308; forceNewPlan=false, retain=false 2024-12-03T22:59:24,527 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=d9c7e273746924274193a9471bc0898e, regionState=OPENING, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:59:24,532 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d9c7e273746924274193a9471bc0898e, ASSIGN because future has completed 2024-12-03T22:59:24,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9c7e273746924274193a9471bc0898e, server=ed8014902436,36731,1733266689308}] 2024-12-03T22:59:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T22:59:24,700 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. 2024-12-03T22:59:24,700 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => d9c7e273746924274193a9471bc0898e, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e.', STARTKEY => '', ENDKEY => ''} 2024-12-03T22:59:24,701 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:24,701 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:59:24,701 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:24,701 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:24,704 INFO [StoreOpener-d9c7e273746924274193a9471bc0898e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:24,706 INFO [StoreOpener-d9c7e273746924274193a9471bc0898e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9c7e273746924274193a9471bc0898e columnFamilyName testFamily 2024-12-03T22:59:24,706 DEBUG [StoreOpener-d9c7e273746924274193a9471bc0898e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:59:24,707 INFO [StoreOpener-d9c7e273746924274193a9471bc0898e-1 {}] regionserver.HStore(327): Store=d9c7e273746924274193a9471bc0898e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T22:59:24,707 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:24,708 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:24,709 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:24,709 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:24,710 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:24,713 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:24,716 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9c7e273746924274193a9471bc0898e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T22:59:24,717 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened d9c7e273746924274193a9471bc0898e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74160275, jitterRate=0.1050742119550705}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T22:59:24,717 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:24,717 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for d9c7e273746924274193a9471bc0898e: Running coprocessor pre-open hook at 1733266764701Writing region info on filesystem at 1733266764701Initializing all the Stores at 1733266764703 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266764703Cleaning up temporary data from old regions at 1733266764710 (+7 ms)Running coprocessor post-open hooks at 1733266764717 (+7 ms)Region opened successfully at 1733266764717 2024-12-03T22:59:24,719 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e., pid=27, masterSystemTime=1733266764688 2024-12-03T22:59:24,721 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. 2024-12-03T22:59:24,721 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. 2024-12-03T22:59:24,721 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=d9c7e273746924274193a9471bc0898e, regionState=OPEN, openSeqNum=2, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:59:24,724 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9c7e273746924274193a9471bc0898e, server=ed8014902436,36731,1733266689308 because future has completed 2024-12-03T22:59:24,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-03T22:59:24,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure d9c7e273746924274193a9471bc0898e, server=ed8014902436,36731,1733266689308 in 192 msec 2024-12-03T22:59:24,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-03T22:59:24,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d9c7e273746924274193a9471bc0898e, ASSIGN in 354 msec 2024-12-03T22:59:24,731 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T22:59:24,731 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266764731"}]},"ts":"1733266764731"} 2024-12-03T22:59:24,734 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-03T22:59:24,735 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T22:59:24,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2100 sec 2024-12-03T22:59:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T22:59:26,555 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-03T22:59:26,555 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-03T22:59:26,556 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:59:26,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-03T22:59:26,566 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:59:26,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-03T22:59:26,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e., hostname=ed8014902436,36731,1733266689308, seqNum=2] 2024-12-03T22:59:26,579 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T22:59:26,580 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-03T22:59:26,582 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T22:59:26,584 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-03T22:59:28,930 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T22:59:28,930 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T22:59:30,206 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T22:59:30,238 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-03T22:59:32,590 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e., hostname=ed8014902436,36731,1733266689308, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e., hostname=ed8014902436,36731,1733266689308, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ed8014902436:36731 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6006ms, rpcTimeout=5999ms 2024-12-03T22:59:32,590 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e., hostname=ed8014902436,36731,1733266689308, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6006ms, rpcTimeout=5999ms 2024-12-03T22:59:32,590 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e., hostname=ed8014902436,36731,1733266689308, seqNum=2 from cache 2024-12-03T22:59:32,605 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=557 (was 605), ProcessCount=11 (was 11), AvailableMemoryMB=5650 (was 5663) 2024-12-03T22:59:32,615 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=557, ProcessCount=11, AvailableMemoryMB=5649 2024-12-03T22:59:32,618 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T22:59:32,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-03T22:59:32,622 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T22:59:32,622 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:59:32,622 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-03T22:59:32,624 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T22:59:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T22:59:32,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741851_1027 (size=407) 2024-12-03T22:59:33,038 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d69b9fe6349586f8b494bb97e25cd10c, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58 2024-12-03T22:59:33,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741852_1028 (size=58) 2024-12-03T22:59:33,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-12-03T22:59:33,452 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:59:33,452 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing d69b9fe6349586f8b494bb97e25cd10c, disabling compactions & flushes 2024-12-03T22:59:33,452 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. 2024-12-03T22:59:33,452 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. 2024-12-03T22:59:33,452 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. after waiting 0 ms 2024-12-03T22:59:33,452 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. 2024-12-03T22:59:33,453 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. 2024-12-03T22:59:33,453 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for d69b9fe6349586f8b494bb97e25cd10c: Waiting for close lock at 1733266773452Disabling compacts and flushes for region at 1733266773452Disabling writes for close at 1733266773452Writing region close event to WAL at 1733266773453 (+1 ms)Closed at 1733266773453 2024-12-03T22:59:33,456 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T22:59:33,457 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733266773456"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733266773456"}]},"ts":"1733266773456"} 2024-12-03T22:59:33,462 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T22:59:33,464 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T22:59:33,464 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266773464"}]},"ts":"1733266773464"} 2024-12-03T22:59:33,505 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-03T22:59:33,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=d69b9fe6349586f8b494bb97e25cd10c, ASSIGN}] 2024-12-03T22:59:33,507 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=d69b9fe6349586f8b494bb97e25cd10c, ASSIGN 2024-12-03T22:59:33,509 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=d69b9fe6349586f8b494bb97e25cd10c, ASSIGN; state=OFFLINE, location=ed8014902436,36731,1733266689308; forceNewPlan=false, retain=false 2024-12-03T22:59:33,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T22:59:33,660 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d69b9fe6349586f8b494bb97e25cd10c, regionState=OPENING, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:59:33,664 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=d69b9fe6349586f8b494bb97e25cd10c, ASSIGN because future has completed 2024-12-03T22:59:33,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d69b9fe6349586f8b494bb97e25cd10c, server=ed8014902436,36731,1733266689308}] 2024-12-03T22:59:33,831 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. 2024-12-03T22:59:33,832 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => d69b9fe6349586f8b494bb97e25cd10c, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c.', STARTKEY => '', ENDKEY => ''} 2024-12-03T22:59:33,833 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext d69b9fe6349586f8b494bb97e25cd10c 2024-12-03T22:59:33,833 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:59:33,833 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for d69b9fe6349586f8b494bb97e25cd10c 2024-12-03T22:59:33,833 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for d69b9fe6349586f8b494bb97e25cd10c 2024-12-03T22:59:33,836 INFO [StoreOpener-d69b9fe6349586f8b494bb97e25cd10c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d69b9fe6349586f8b494bb97e25cd10c 2024-12-03T22:59:33,839 INFO [StoreOpener-d69b9fe6349586f8b494bb97e25cd10c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d69b9fe6349586f8b494bb97e25cd10c columnFamilyName testFamily 2024-12-03T22:59:33,839 DEBUG [StoreOpener-d69b9fe6349586f8b494bb97e25cd10c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:59:33,840 INFO [StoreOpener-d69b9fe6349586f8b494bb97e25cd10c-1 {}] regionserver.HStore(327): Store=d69b9fe6349586f8b494bb97e25cd10c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T22:59:33,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for d69b9fe6349586f8b494bb97e25cd10c 2024-12-03T22:59:33,842 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNext/d69b9fe6349586f8b494bb97e25cd10c 2024-12-03T22:59:33,843 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNext/d69b9fe6349586f8b494bb97e25cd10c 2024-12-03T22:59:33,844 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for d69b9fe6349586f8b494bb97e25cd10c 2024-12-03T22:59:33,844 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for d69b9fe6349586f8b494bb97e25cd10c 2024-12-03T22:59:33,847 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for d69b9fe6349586f8b494bb97e25cd10c 2024-12-03T22:59:33,851 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNext/d69b9fe6349586f8b494bb97e25cd10c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T22:59:33,852 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened d69b9fe6349586f8b494bb97e25cd10c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69972884, jitterRate=0.042677223682403564}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T22:59:33,852 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d69b9fe6349586f8b494bb97e25cd10c 2024-12-03T22:59:33,853 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for d69b9fe6349586f8b494bb97e25cd10c: Running coprocessor pre-open hook at 1733266773833Writing region info on filesystem at 1733266773833Initializing all the Stores at 1733266773835 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266773836 (+1 ms)Cleaning up temporary data from old regions at 1733266773844 (+8 ms)Running coprocessor post-open hooks at 1733266773852 (+8 ms)Region opened successfully at 1733266773853 (+1 ms) 2024-12-03T22:59:33,854 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c., pid=30, masterSystemTime=1733266773820 2024-12-03T22:59:33,857 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. 2024-12-03T22:59:33,857 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. 2024-12-03T22:59:33,858 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d69b9fe6349586f8b494bb97e25cd10c, regionState=OPEN, openSeqNum=2, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:59:33,861 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d69b9fe6349586f8b494bb97e25cd10c, server=ed8014902436,36731,1733266689308 because future has completed 2024-12-03T22:59:33,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-03T22:59:33,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure d69b9fe6349586f8b494bb97e25cd10c, server=ed8014902436,36731,1733266689308 in 198 msec 2024-12-03T22:59:33,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-03T22:59:33,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=d69b9fe6349586f8b494bb97e25cd10c, ASSIGN in 361 msec 2024-12-03T22:59:33,870 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T22:59:33,870 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266773870"}]},"ts":"1733266773870"} 2024-12-03T22:59:33,872 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-03T22:59:33,873 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T22:59:33,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2550 sec 2024-12-03T22:59:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T22:59:35,655 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-03T22:59:35,655 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-03T22:59:35,655 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:59:35,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-03T22:59:35,666 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:59:35,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-03T22:59:35,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c., hostname=ed8014902436,36731,1733266689308, seqNum=2] 2024-12-03T22:59:35,680 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T22:59:35,680 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-03T22:59:35,684 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T22:59:35,686 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T22:59:35,687 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T22:59:35,688 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-03T22:59:36,241 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-03T22:59:37,539 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T22:59:39,338 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T22:59:39,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-03T22:59:39,372 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-03T22:59:41,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36731 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '699721224162377780' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T22:59:41,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36731 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:34294 deadline: 1733266787690, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '699721224162377780' does not exist 2024-12-03T22:59:41,693 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to ed8014902436,36731,1733266689308 for closing scanner id = 699721224162377780 for d69b9fe6349586f8b494bb97e25cd10c of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '699721224162377780' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T22:59:41,700 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=478 (was 557), ProcessCount=11 (was 11), AvailableMemoryMB=5653 (was 5649) - AvailableMemoryMB LEAK? - 2024-12-03T22:59:41,707 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=478, ProcessCount=11, AvailableMemoryMB=5652 2024-12-03T22:59:41,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T22:59:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-03T22:59:41,711 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T22:59:41,711 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:59:41,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-03T22:59:41,713 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T22:59:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T22:59:41,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741853_1029 (size=423) 2024-12-03T22:59:42,127 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0ceb224e1117f295bbf34dd963d59fe3, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58 2024-12-03T22:59:42,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741854_1030 (size=74) 2024-12-03T22:59:42,143 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:59:42,143 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 0ceb224e1117f295bbf34dd963d59fe3, disabling compactions & flushes 2024-12-03T22:59:42,143 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. 2024-12-03T22:59:42,143 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. 2024-12-03T22:59:42,143 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. after waiting 0 ms 2024-12-03T22:59:42,143 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. 2024-12-03T22:59:42,143 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. 2024-12-03T22:59:42,143 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0ceb224e1117f295bbf34dd963d59fe3: Waiting for close lock at 1733266782143Disabling compacts and flushes for region at 1733266782143Disabling writes for close at 1733266782143Writing region close event to WAL at 1733266782143Closed at 1733266782143 2024-12-03T22:59:42,145 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T22:59:42,145 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733266782145"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733266782145"}]},"ts":"1733266782145"} 2024-12-03T22:59:42,148 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T22:59:42,149 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T22:59:42,149 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266782149"}]},"ts":"1733266782149"} 2024-12-03T22:59:42,151 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-03T22:59:42,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0ceb224e1117f295bbf34dd963d59fe3, ASSIGN}] 2024-12-03T22:59:42,153 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0ceb224e1117f295bbf34dd963d59fe3, ASSIGN 2024-12-03T22:59:42,154 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0ceb224e1117f295bbf34dd963d59fe3, ASSIGN; state=OFFLINE, location=ed8014902436,36731,1733266689308; forceNewPlan=false, retain=false 2024-12-03T22:59:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-12-03T22:59:42,306 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=0ceb224e1117f295bbf34dd963d59fe3, regionState=OPENING, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:59:42,312 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0ceb224e1117f295bbf34dd963d59fe3, ASSIGN because future has completed 2024-12-03T22:59:42,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ceb224e1117f295bbf34dd963d59fe3, server=ed8014902436,36731,1733266689308}] 2024-12-03T22:59:42,477 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. 2024-12-03T22:59:42,477 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ceb224e1117f295bbf34dd963d59fe3, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3.', STARTKEY => '', ENDKEY => ''} 2024-12-03T22:59:42,478 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 0ceb224e1117f295bbf34dd963d59fe3 2024-12-03T22:59:42,478 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T22:59:42,478 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 0ceb224e1117f295bbf34dd963d59fe3 2024-12-03T22:59:42,478 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 0ceb224e1117f295bbf34dd963d59fe3 2024-12-03T22:59:42,481 INFO [StoreOpener-0ceb224e1117f295bbf34dd963d59fe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0ceb224e1117f295bbf34dd963d59fe3 2024-12-03T22:59:42,484 INFO [StoreOpener-0ceb224e1117f295bbf34dd963d59fe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ceb224e1117f295bbf34dd963d59fe3 columnFamilyName testFamily 2024-12-03T22:59:42,484 DEBUG [StoreOpener-0ceb224e1117f295bbf34dd963d59fe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T22:59:42,485 INFO [StoreOpener-0ceb224e1117f295bbf34dd963d59fe3-1 {}] regionserver.HStore(327): Store=0ceb224e1117f295bbf34dd963d59fe3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T22:59:42,485 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 0ceb224e1117f295bbf34dd963d59fe3 2024-12-03T22:59:42,487 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ceb224e1117f295bbf34dd963d59fe3 2024-12-03T22:59:42,487 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ceb224e1117f295bbf34dd963d59fe3 2024-12-03T22:59:42,488 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 0ceb224e1117f295bbf34dd963d59fe3 2024-12-03T22:59:42,488 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 0ceb224e1117f295bbf34dd963d59fe3 2024-12-03T22:59:42,491 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 0ceb224e1117f295bbf34dd963d59fe3 2024-12-03T22:59:42,494 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ceb224e1117f295bbf34dd963d59fe3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T22:59:42,495 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 0ceb224e1117f295bbf34dd963d59fe3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74262903, jitterRate=0.10660348832607269}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T22:59:42,495 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ceb224e1117f295bbf34dd963d59fe3 2024-12-03T22:59:42,495 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 0ceb224e1117f295bbf34dd963d59fe3: Running coprocessor pre-open hook at 1733266782478Writing region info on filesystem at 1733266782478Initializing all the Stores at 1733266782480 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733266782480Cleaning up temporary data from old regions at 1733266782488 (+8 ms)Running coprocessor post-open hooks at 1733266782495 (+7 ms)Region opened successfully at 1733266782495 2024-12-03T22:59:42,496 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3., pid=33, masterSystemTime=1733266782466 2024-12-03T22:59:42,499 DEBUG [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. 2024-12-03T22:59:42,499 INFO [RS_OPEN_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. 2024-12-03T22:59:42,500 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=0ceb224e1117f295bbf34dd963d59fe3, regionState=OPEN, openSeqNum=2, regionLocation=ed8014902436,36731,1733266689308 2024-12-03T22:59:42,502 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ceb224e1117f295bbf34dd963d59fe3, server=ed8014902436,36731,1733266689308 because future has completed 2024-12-03T22:59:42,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-03T22:59:42,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ceb224e1117f295bbf34dd963d59fe3, server=ed8014902436,36731,1733266689308 in 190 msec 2024-12-03T22:59:42,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-03T22:59:42,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0ceb224e1117f295bbf34dd963d59fe3, ASSIGN in 354 msec 2024-12-03T22:59:42,509 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T22:59:42,509 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733266782509"}]},"ts":"1733266782509"} 2024-12-03T22:59:42,511 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-03T22:59:42,512 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T22:59:42,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 803 msec 2024-12-03T22:59:42,623 DEBUG [master/ed8014902436:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T22:59:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T22:59:42,734 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-03T22:59:42,734 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-03T22:59:42,735 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:59:42,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-03T22:59:42,746 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T22:59:42,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-03T22:59:42,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3., hostname=ed8014902436,36731,1733266689308, seqNum=2] 2024-12-03T22:59:42,757 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T22:59:42,757 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-03T22:59:42,760 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T22:59:42,761 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T22:59:43,769 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3., hostname=ed8014902436,36731,1733266689308, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3., hostname=ed8014902436,36731,1733266689308, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ed8014902436:36731 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-12-03T22:59:43,769 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3., hostname=ed8014902436,36731,1733266689308, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-12-03T22:59:43,770 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3., hostname=ed8014902436,36731,1733266689308, seqNum=2 from cache 2024-12-03T22:59:44,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-12-03T22:59:44,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3., hostname=ed8014902436,36731,1733266689308, seqNum=2] 2024-12-03T22:59:44,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T22:59:45,770 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3., hostname=ed8014902436,36731,1733266689308, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3., hostname=ed8014902436,36731,1733266689308, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=ed8014902436:36731 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=982ms, rpcTimeout=973ms 2024-12-03T22:59:45,770 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3., hostname=ed8014902436,36731,1733266689308, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=982ms, rpcTimeout=973ms 2024-12-03T22:59:45,770 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3., hostname=ed8014902436,36731,1733266689308, seqNum=2 from cache 2024-12-03T22:59:45,784 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=456 (was 478), ProcessCount=11 (was 11), AvailableMemoryMB=5645 (was 5652) 2024-12-03T22:59:45,784 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T22:59:45,784 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T22:59:45,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T22:59:45,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T22:59:45,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T22:59:45,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T22:59:45,789 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T22:59:45,789 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T22:59:45,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T22:59:45,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T22:59:45,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T22:59:45,790 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T22:59:45,790 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1546128933, stopped=false 2024-12-03T22:59:45,790 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ed8014902436,44997,1733266688559 2024-12-03T22:59:45,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T22:59:45,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T22:59:45,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:59:45,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:59:45,878 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T22:59:45,880 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T22:59:45,880 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T22:59:45,880 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T22:59:45,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T22:59:45,881 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T22:59:45,881 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ed8014902436,36731,1733266689308' ***** 2024-12-03T22:59:45,881 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T22:59:45,882 INFO [RS:0;ed8014902436:36731 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T22:59:45,882 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T22:59:45,882 INFO [RS:0;ed8014902436:36731 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T22:59:45,882 INFO [RS:0;ed8014902436:36731 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T22:59:45,883 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(3091): Received CLOSE for 0ceb224e1117f295bbf34dd963d59fe3 2024-12-03T22:59:45,883 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(3091): Received CLOSE for 46d22f3dadfd475287722d11d378de03 2024-12-03T22:59:45,883 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(3091): Received CLOSE for d69b9fe6349586f8b494bb97e25cd10c 2024-12-03T22:59:45,883 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(3091): Received CLOSE for e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:45,883 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(3091): Received CLOSE for bbb748d5bd9330e44a775a0d924c9164 2024-12-03T22:59:45,883 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(3091): Received CLOSE for a4bff265cf897b5304f15f9f43579d6c 2024-12-03T22:59:45,883 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(3091): Received CLOSE for d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:45,884 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(3091): Received CLOSE for 897bed5ed2ecae70c69cd2703f2323f9 2024-12-03T22:59:45,884 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(3091): Received CLOSE for 3cd9eb5ad6480573474415850e7c0740 2024-12-03T22:59:45,884 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(3091): Received CLOSE for 78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:59:45,884 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0ceb224e1117f295bbf34dd963d59fe3, disabling compactions & flushes 2024-12-03T22:59:45,884 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. 2024-12-03T22:59:45,884 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(959): stopping server ed8014902436,36731,1733266689308 2024-12-03T22:59:45,884 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. 2024-12-03T22:59:45,884 INFO [RS:0;ed8014902436:36731 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T22:59:45,884 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. after waiting 0 ms 2024-12-03T22:59:45,884 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. 2024-12-03T22:59:45,884 INFO [RS:0;ed8014902436:36731 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ed8014902436:36731. 2024-12-03T22:59:45,884 DEBUG [RS:0;ed8014902436:36731 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T22:59:45,884 DEBUG [RS:0;ed8014902436:36731 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T22:59:45,884 INFO [RS:0;ed8014902436:36731 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T22:59:45,885 INFO [RS:0;ed8014902436:36731 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T22:59:45,885 INFO [RS:0;ed8014902436:36731 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T22:59:45,885 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T22:59:45,885 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-03T22:59:45,885 DEBUG [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T22:59:45,886 INFO [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T22:59:45,886 DEBUG [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T22:59:45,886 DEBUG [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T22:59:45,886 DEBUG [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T22:59:45,886 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 0ceb224e1117f295bbf34dd963d59fe3, 1588230740, 3cd9eb5ad6480573474415850e7c0740, 46d22f3dadfd475287722d11d378de03, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, a4bff265cf897b5304f15f9f43579d6c, bbb748d5bd9330e44a775a0d924c9164, d69b9fe6349586f8b494bb97e25cd10c, d9c7e273746924274193a9471bc0898e, e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:45,888 INFO [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-03T22:59:45,888 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0ceb224e1117f295bbf34dd963d59fe3 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T22:59:45,937 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ceb224e1117f295bbf34dd963d59fe3/.tmp/testFamily/231329acea00449885bf6b517137c1fc is 61, key is row-0/testFamily:testQualifier/1733266782749/Put/seqid=0 2024-12-03T22:59:45,939 DEBUG [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/.tmp/info/7173a774bc8e45fba5c1fff3d8e2de7c is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164./info:regioninfo/1733266724006/Put/seqid=0 2024-12-03T22:59:45,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741855_1031 (size=5049) 2024-12-03T22:59:45,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741856_1032 (size=20564) 2024-12-03T22:59:46,086 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 0ceb224e1117f295bbf34dd963d59fe3, 1588230740, 3cd9eb5ad6480573474415850e7c0740, 46d22f3dadfd475287722d11d378de03, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, a4bff265cf897b5304f15f9f43579d6c, bbb748d5bd9330e44a775a0d924c9164, d69b9fe6349586f8b494bb97e25cd10c, d9c7e273746924274193a9471bc0898e, e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:46,287 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 0ceb224e1117f295bbf34dd963d59fe3, 1588230740, 3cd9eb5ad6480573474415850e7c0740, 46d22f3dadfd475287722d11d378de03, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, a4bff265cf897b5304f15f9f43579d6c, bbb748d5bd9330e44a775a0d924c9164, d69b9fe6349586f8b494bb97e25cd10c, d9c7e273746924274193a9471bc0898e, e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:46,290 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-03T22:59:46,351 INFO [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/.tmp/info/7173a774bc8e45fba5c1fff3d8e2de7c 2024-12-03T22:59:46,351 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ceb224e1117f295bbf34dd963d59fe3/.tmp/testFamily/231329acea00449885bf6b517137c1fc 2024-12-03T22:59:46,393 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ceb224e1117f295bbf34dd963d59fe3/.tmp/testFamily/231329acea00449885bf6b517137c1fc as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ceb224e1117f295bbf34dd963d59fe3/testFamily/231329acea00449885bf6b517137c1fc 2024-12-03T22:59:46,401 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ceb224e1117f295bbf34dd963d59fe3/testFamily/231329acea00449885bf6b517137c1fc, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T22:59:46,408 DEBUG [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/.tmp/ns/df66874e7383488a9e1cdcd3ad62de97 is 43, key is default/ns:d/1733266722538/Put/seqid=0 2024-12-03T22:59:46,410 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0ceb224e1117f295bbf34dd963d59fe3 in 521ms, sequenceid=8, compaction requested=false 2024-12-03T22:59:46,410 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-03T22:59:46,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741857_1033 (size=5153) 2024-12-03T22:59:46,417 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ceb224e1117f295bbf34dd963d59fe3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T22:59:46,419 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. 2024-12-03T22:59:46,419 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0ceb224e1117f295bbf34dd963d59fe3: Waiting for close lock at 1733266785883Running coprocessor pre-close hooks at 1733266785884 (+1 ms)Disabling compacts and flushes for region at 1733266785884Disabling writes for close at 1733266785884Obtaining lock to block concurrent updates at 1733266785888 (+4 ms)Preparing flush snapshotting stores in 0ceb224e1117f295bbf34dd963d59fe3 at 1733266785888Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733266785894 (+6 ms)Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. at 1733266785895 (+1 ms)Flushing 0ceb224e1117f295bbf34dd963d59fe3/testFamily: creating writer at 1733266785897 (+2 ms)Flushing 0ceb224e1117f295bbf34dd963d59fe3/testFamily: appending metadata at 1733266785931 (+34 ms)Flushing 0ceb224e1117f295bbf34dd963d59fe3/testFamily: closing flushed file at 1733266785933 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fee01e5: reopening flushed file at 1733266786392 (+459 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0ceb224e1117f295bbf34dd963d59fe3 in 521ms, sequenceid=8, compaction requested=false at 1733266786410 (+18 ms)Writing region close event to WAL at 1733266786412 (+2 ms)Running coprocessor post-close hooks at 1733266786418 (+6 ms)Closed at 1733266786419 (+1 ms) 2024-12-03T22:59:46,420 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733266781709.0ceb224e1117f295bbf34dd963d59fe3. 2024-12-03T22:59:46,420 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 46d22f3dadfd475287722d11d378de03, disabling compactions & flushes 2024-12-03T22:59:46,420 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. 2024-12-03T22:59:46,420 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. 2024-12-03T22:59:46,420 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. after waiting 0 ms 2024-12-03T22:59:46,421 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. 2024-12-03T22:59:46,421 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 46d22f3dadfd475287722d11d378de03 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T22:59:46,437 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNextAsync/46d22f3dadfd475287722d11d378de03/.tmp/testFamily/062659cc4ecb4efd937c5c2922783237 is 61, key is row-0/testFamily:testQualifier/1733266754365/Put/seqid=0 2024-12-03T22:59:46,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741858_1034 (size=5049) 2024-12-03T22:59:46,487 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3cd9eb5ad6480573474415850e7c0740, 46d22f3dadfd475287722d11d378de03, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, a4bff265cf897b5304f15f9f43579d6c, bbb748d5bd9330e44a775a0d924c9164, d69b9fe6349586f8b494bb97e25cd10c, d9c7e273746924274193a9471bc0898e, e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:46,687 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3cd9eb5ad6480573474415850e7c0740, 46d22f3dadfd475287722d11d378de03, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, a4bff265cf897b5304f15f9f43579d6c, bbb748d5bd9330e44a775a0d924c9164, d69b9fe6349586f8b494bb97e25cd10c, d9c7e273746924274193a9471bc0898e, e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:46,817 INFO [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/.tmp/ns/df66874e7383488a9e1cdcd3ad62de97 2024-12-03T22:59:46,843 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNextAsync/46d22f3dadfd475287722d11d378de03/.tmp/testFamily/062659cc4ecb4efd937c5c2922783237 2024-12-03T22:59:46,847 DEBUG [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/.tmp/table/aaf62843ca064bb18a685a64c76cd9c9 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733266724021/Put/seqid=0 2024-12-03T22:59:46,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741859_1035 (size=6699) 2024-12-03T22:59:46,853 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNextAsync/46d22f3dadfd475287722d11d378de03/.tmp/testFamily/062659cc4ecb4efd937c5c2922783237 as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNextAsync/46d22f3dadfd475287722d11d378de03/testFamily/062659cc4ecb4efd937c5c2922783237 2024-12-03T22:59:46,862 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNextAsync/46d22f3dadfd475287722d11d378de03/testFamily/062659cc4ecb4efd937c5c2922783237, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T22:59:46,863 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 46d22f3dadfd475287722d11d378de03 in 442ms, sequenceid=8, compaction requested=false 2024-12-03T22:59:46,868 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNextAsync/46d22f3dadfd475287722d11d378de03/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T22:59:46,868 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. 2024-12-03T22:59:46,869 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 46d22f3dadfd475287722d11d378de03: Waiting for close lock at 1733266786420Running coprocessor pre-close hooks at 1733266786420Disabling compacts and flushes for region at 1733266786420Disabling writes for close at 1733266786421 (+1 ms)Obtaining lock to block concurrent updates at 1733266786421Preparing flush snapshotting stores in 46d22f3dadfd475287722d11d378de03 at 1733266786421Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733266786421Flushing stores of testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. at 1733266786422 (+1 ms)Flushing 46d22f3dadfd475287722d11d378de03/testFamily: creating writer at 1733266786422Flushing 46d22f3dadfd475287722d11d378de03/testFamily: appending metadata at 1733266786437 (+15 ms)Flushing 46d22f3dadfd475287722d11d378de03/testFamily: closing flushed file at 1733266786437Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76501a26: reopening flushed file at 1733266786852 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 46d22f3dadfd475287722d11d378de03 in 442ms, sequenceid=8, compaction requested=false at 1733266786863 (+11 ms)Writing region close event to WAL at 1733266786863Running coprocessor post-close hooks at 1733266786868 (+5 ms)Closed at 1733266786868 2024-12-03T22:59:46,869 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733266751326.46d22f3dadfd475287722d11d378de03. 2024-12-03T22:59:46,869 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d69b9fe6349586f8b494bb97e25cd10c, disabling compactions & flushes 2024-12-03T22:59:46,869 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. 2024-12-03T22:59:46,869 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. 2024-12-03T22:59:46,869 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. after waiting 0 ms 2024-12-03T22:59:46,869 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. 2024-12-03T22:59:46,869 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d69b9fe6349586f8b494bb97e25cd10c 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T22:59:46,885 INFO [regionserver/ed8014902436:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T22:59:46,886 INFO [regionserver/ed8014902436:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T22:59:46,886 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNext/d69b9fe6349586f8b494bb97e25cd10c/.tmp/testFamily/337533f0f8934498b5201e7f04212703 is 61, key is row-0/testFamily:testQualifier/1733266775670/Put/seqid=0 2024-12-03T22:59:46,888 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-12-03T22:59:46,888 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1325): Online Regions={d69b9fe6349586f8b494bb97e25cd10c=hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c., e048d04e396089b38cef2a0e42e3fdf4=testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4., bbb748d5bd9330e44a775a0d924c9164=testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164., 1588230740=hbase:meta,,1.1588230740, a4bff265cf897b5304f15f9f43579d6c=testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c., d9c7e273746924274193a9471bc0898e=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e., 897bed5ed2ecae70c69cd2703f2323f9=hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9., 3cd9eb5ad6480573474415850e7c0740=testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740., 78246d6932357f4d3bdead6e1777c4c5=hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5.} 2024-12-03T22:59:46,888 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, a4bff265cf897b5304f15f9f43579d6c, bbb748d5bd9330e44a775a0d924c9164, d69b9fe6349586f8b494bb97e25cd10c, d9c7e273746924274193a9471bc0898e, e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:46,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741860_1036 (size=5049) 2024-12-03T22:59:47,088 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, a4bff265cf897b5304f15f9f43579d6c, bbb748d5bd9330e44a775a0d924c9164, d69b9fe6349586f8b494bb97e25cd10c, d9c7e273746924274193a9471bc0898e, e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:47,253 INFO [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/.tmp/table/aaf62843ca064bb18a685a64c76cd9c9 2024-12-03T22:59:47,265 DEBUG [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/.tmp/info/7173a774bc8e45fba5c1fff3d8e2de7c as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/info/7173a774bc8e45fba5c1fff3d8e2de7c 2024-12-03T22:59:47,275 INFO [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/info/7173a774bc8e45fba5c1fff3d8e2de7c, entries=100, sequenceid=56, filesize=20.1 K 2024-12-03T22:59:47,277 DEBUG [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/.tmp/ns/df66874e7383488a9e1cdcd3ad62de97 as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/ns/df66874e7383488a9e1cdcd3ad62de97 2024-12-03T22:59:47,285 INFO [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/ns/df66874e7383488a9e1cdcd3ad62de97, entries=2, sequenceid=56, filesize=5.0 K 2024-12-03T22:59:47,287 DEBUG [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/.tmp/table/aaf62843ca064bb18a685a64c76cd9c9 as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/table/aaf62843ca064bb18a685a64c76cd9c9 2024-12-03T22:59:47,289 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, a4bff265cf897b5304f15f9f43579d6c, bbb748d5bd9330e44a775a0d924c9164, d69b9fe6349586f8b494bb97e25cd10c, d9c7e273746924274193a9471bc0898e, e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:47,291 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNext/d69b9fe6349586f8b494bb97e25cd10c/.tmp/testFamily/337533f0f8934498b5201e7f04212703 2024-12-03T22:59:47,296 INFO [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/table/aaf62843ca064bb18a685a64c76cd9c9, entries=20, sequenceid=56, filesize=6.5 K 2024-12-03T22:59:47,297 INFO [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1411ms, sequenceid=56, compaction requested=false 2024-12-03T22:59:47,299 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNext/d69b9fe6349586f8b494bb97e25cd10c/.tmp/testFamily/337533f0f8934498b5201e7f04212703 as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNext/d69b9fe6349586f8b494bb97e25cd10c/testFamily/337533f0f8934498b5201e7f04212703 2024-12-03T22:59:47,303 DEBUG [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-03T22:59:47,304 DEBUG [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T22:59:47,305 INFO [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T22:59:47,305 DEBUG [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733266785885Running coprocessor pre-close hooks at 1733266785885Disabling compacts and flushes for region at 1733266785885Disabling writes for close at 1733266785886 (+1 ms)Obtaining lock to block concurrent updates at 1733266785888 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1733266785888Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733266785894 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733266785895 (+1 ms)Flushing 1588230740/info: creating writer at 1733266785897 (+2 ms)Flushing 1588230740/info: appending metadata at 1733266785936 (+39 ms)Flushing 1588230740/info: closing flushed file at 1733266785936Flushing 1588230740/ns: creating writer at 1733266786392 (+456 ms)Flushing 1588230740/ns: appending metadata at 1733266786407 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733266786407Flushing 1588230740/table: creating writer at 1733266786829 (+422 ms)Flushing 1588230740/table: appending metadata at 1733266786846 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733266786846Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4de08721: reopening flushed file at 1733266787264 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46651b8a: reopening flushed file at 1733266787276 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f023f8a: reopening flushed file at 1733266787286 (+10 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1411ms, sequenceid=56, compaction requested=false at 1733266787297 (+11 ms)Writing region close event to WAL at 1733266787299 (+2 ms)Running coprocessor post-close hooks at 1733266787304 (+5 ms)Closed at 1733266787304 2024-12-03T22:59:47,305 DEBUG [RS_CLOSE_META-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T22:59:47,307 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNext/d69b9fe6349586f8b494bb97e25cd10c/testFamily/337533f0f8934498b5201e7f04212703, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T22:59:47,308 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d69b9fe6349586f8b494bb97e25cd10c in 439ms, sequenceid=8, compaction requested=false 2024-12-03T22:59:47,313 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNext/d69b9fe6349586f8b494bb97e25cd10c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T22:59:47,314 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. 2024-12-03T22:59:47,314 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d69b9fe6349586f8b494bb97e25cd10c: Waiting for close lock at 1733266786869Running coprocessor pre-close hooks at 1733266786869Disabling compacts and flushes for region at 1733266786869Disabling writes for close at 1733266786869Obtaining lock to block concurrent updates at 1733266786869Preparing flush snapshotting stores in d69b9fe6349586f8b494bb97e25cd10c at 1733266786869Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733266786870 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. at 1733266786870Flushing d69b9fe6349586f8b494bb97e25cd10c/testFamily: creating writer at 1733266786870Flushing d69b9fe6349586f8b494bb97e25cd10c/testFamily: appending metadata at 1733266786885 (+15 ms)Flushing d69b9fe6349586f8b494bb97e25cd10c/testFamily: closing flushed file at 1733266786885Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70f93dac: reopening flushed file at 1733266787298 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d69b9fe6349586f8b494bb97e25cd10c in 439ms, sequenceid=8, compaction requested=false at 1733266787308 (+10 ms)Writing region close event to WAL at 1733266787309 (+1 ms)Running coprocessor post-close hooks at 1733266787313 (+4 ms)Closed at 1733266787313 2024-12-03T22:59:47,314 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733266772617.d69b9fe6349586f8b494bb97e25cd10c. 2024-12-03T22:59:47,314 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e048d04e396089b38cef2a0e42e3fdf4, disabling compactions & flushes 2024-12-03T22:59:47,314 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. 2024-12-03T22:59:47,314 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. 2024-12-03T22:59:47,314 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. after waiting 0 ms 2024-12-03T22:59:47,314 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. 2024-12-03T22:59:47,314 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e048d04e396089b38cef2a0e42e3fdf4 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T22:59:47,332 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNext/e048d04e396089b38cef2a0e42e3fdf4/.tmp/testFamily/0fbe8282dafe4953ac164f4070088271 is 61, key is row-0/testFamily:testQualifier/1733266760470/Put/seqid=0 2024-12-03T22:59:47,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741861_1037 (size=5049) 2024-12-03T22:59:47,489 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, a4bff265cf897b5304f15f9f43579d6c, bbb748d5bd9330e44a775a0d924c9164, d9c7e273746924274193a9471bc0898e, e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:47,690 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, a4bff265cf897b5304f15f9f43579d6c, bbb748d5bd9330e44a775a0d924c9164, d9c7e273746924274193a9471bc0898e, e048d04e396089b38cef2a0e42e3fdf4 2024-12-03T22:59:47,740 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNext/e048d04e396089b38cef2a0e42e3fdf4/.tmp/testFamily/0fbe8282dafe4953ac164f4070088271 2024-12-03T22:59:47,752 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNext/e048d04e396089b38cef2a0e42e3fdf4/.tmp/testFamily/0fbe8282dafe4953ac164f4070088271 as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNext/e048d04e396089b38cef2a0e42e3fdf4/testFamily/0fbe8282dafe4953ac164f4070088271 2024-12-03T22:59:47,758 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNext/e048d04e396089b38cef2a0e42e3fdf4/testFamily/0fbe8282dafe4953ac164f4070088271, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T22:59:47,759 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e048d04e396089b38cef2a0e42e3fdf4 in 445ms, sequenceid=8, compaction requested=false 2024-12-03T22:59:47,764 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnNext/e048d04e396089b38cef2a0e42e3fdf4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T22:59:47,764 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. 2024-12-03T22:59:47,764 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e048d04e396089b38cef2a0e42e3fdf4: Waiting for close lock at 1733266787314Running coprocessor pre-close hooks at 1733266787314Disabling compacts and flushes for region at 1733266787314Disabling writes for close at 1733266787314Obtaining lock to block concurrent updates at 1733266787314Preparing flush snapshotting stores in e048d04e396089b38cef2a0e42e3fdf4 at 1733266787314Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733266787314Flushing stores of testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. at 1733266787315 (+1 ms)Flushing e048d04e396089b38cef2a0e42e3fdf4/testFamily: creating writer at 1733266787315Flushing e048d04e396089b38cef2a0e42e3fdf4/testFamily: appending metadata at 1733266787332 (+17 ms)Flushing e048d04e396089b38cef2a0e42e3fdf4/testFamily: closing flushed file at 1733266787332Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bf9aa05: reopening flushed file at 1733266787751 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e048d04e396089b38cef2a0e42e3fdf4 in 445ms, sequenceid=8, compaction requested=false at 1733266787759 (+8 ms)Writing region close event to WAL at 1733266787760 (+1 ms)Running coprocessor post-close hooks at 1733266787764 (+4 ms)Closed at 1733266787764 2024-12-03T22:59:47,765 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733266757416.e048d04e396089b38cef2a0e42e3fdf4. 2024-12-03T22:59:47,765 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bbb748d5bd9330e44a775a0d924c9164, disabling compactions & flushes 2024-12-03T22:59:47,765 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. 2024-12-03T22:59:47,765 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. 2024-12-03T22:59:47,765 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. after waiting 0 ms 2024-12-03T22:59:47,765 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. 2024-12-03T22:59:47,765 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing bbb748d5bd9330e44a775a0d924c9164 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T22:59:47,779 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bbb748d5bd9330e44a775a0d924c9164/.tmp/testFamily/48b098ccd3f54401b22add4002208b1f is 61, key is row-0/testFamily:testQualifier/1733266725862/Put/seqid=0 2024-12-03T22:59:47,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741862_1038 (size=5049) 2024-12-03T22:59:47,890 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-12-03T22:59:47,890 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1325): Online Regions={bbb748d5bd9330e44a775a0d924c9164=testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164., a4bff265cf897b5304f15f9f43579d6c=testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c., d9c7e273746924274193a9471bc0898e=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e., 897bed5ed2ecae70c69cd2703f2323f9=hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9., 3cd9eb5ad6480573474415850e7c0740=testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740., 78246d6932357f4d3bdead6e1777c4c5=hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5.} 2024-12-03T22:59:47,890 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, a4bff265cf897b5304f15f9f43579d6c, bbb748d5bd9330e44a775a0d924c9164, d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:47,982 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T22:59:48,091 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, a4bff265cf897b5304f15f9f43579d6c, bbb748d5bd9330e44a775a0d924c9164, d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:48,185 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bbb748d5bd9330e44a775a0d924c9164/.tmp/testFamily/48b098ccd3f54401b22add4002208b1f 2024-12-03T22:59:48,194 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bbb748d5bd9330e44a775a0d924c9164/.tmp/testFamily/48b098ccd3f54401b22add4002208b1f as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bbb748d5bd9330e44a775a0d924c9164/testFamily/48b098ccd3f54401b22add4002208b1f 2024-12-03T22:59:48,201 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bbb748d5bd9330e44a775a0d924c9164/testFamily/48b098ccd3f54401b22add4002208b1f, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T22:59:48,202 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bbb748d5bd9330e44a775a0d924c9164 in 437ms, sequenceid=8, compaction requested=false 2024-12-03T22:59:48,208 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextExceptionAsync/bbb748d5bd9330e44a775a0d924c9164/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T22:59:48,208 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. 2024-12-03T22:59:48,208 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bbb748d5bd9330e44a775a0d924c9164: Waiting for close lock at 1733266787765Running coprocessor pre-close hooks at 1733266787765Disabling compacts and flushes for region at 1733266787765Disabling writes for close at 1733266787765Obtaining lock to block concurrent updates at 1733266787765Preparing flush snapshotting stores in bbb748d5bd9330e44a775a0d924c9164 at 1733266787765Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733266787765Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. at 1733266787766 (+1 ms)Flushing bbb748d5bd9330e44a775a0d924c9164/testFamily: creating writer at 1733266787766Flushing bbb748d5bd9330e44a775a0d924c9164/testFamily: appending metadata at 1733266787779 (+13 ms)Flushing bbb748d5bd9330e44a775a0d924c9164/testFamily: closing flushed file at 1733266787779Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77742650: reopening flushed file at 1733266788193 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bbb748d5bd9330e44a775a0d924c9164 in 437ms, sequenceid=8, compaction requested=false at 1733266788202 (+9 ms)Writing region close event to WAL at 1733266788203 (+1 ms)Running coprocessor post-close hooks at 1733266788208 (+5 ms)Closed at 1733266788208 2024-12-03T22:59:48,209 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733266722754.bbb748d5bd9330e44a775a0d924c9164. 2024-12-03T22:59:48,209 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a4bff265cf897b5304f15f9f43579d6c, disabling compactions & flushes 2024-12-03T22:59:48,209 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. 2024-12-03T22:59:48,209 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. 2024-12-03T22:59:48,209 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. after waiting 0 ms 2024-12-03T22:59:48,209 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. 2024-12-03T22:59:48,209 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a4bff265cf897b5304f15f9f43579d6c 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T22:59:48,225 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScanner/a4bff265cf897b5304f15f9f43579d6c/.tmp/testFamily/c06cf4aadf634f1ea8f125bb08bd0240 is 61, key is row-0/testFamily:testQualifier/1733266748280/Put/seqid=0 2024-12-03T22:59:48,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741863_1039 (size=5049) 2024-12-03T22:59:48,291 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, a4bff265cf897b5304f15f9f43579d6c, d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:48,491 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, a4bff265cf897b5304f15f9f43579d6c, d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:48,632 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScanner/a4bff265cf897b5304f15f9f43579d6c/.tmp/testFamily/c06cf4aadf634f1ea8f125bb08bd0240 2024-12-03T22:59:48,646 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScanner/a4bff265cf897b5304f15f9f43579d6c/.tmp/testFamily/c06cf4aadf634f1ea8f125bb08bd0240 as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScanner/a4bff265cf897b5304f15f9f43579d6c/testFamily/c06cf4aadf634f1ea8f125bb08bd0240 2024-12-03T22:59:48,655 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScanner/a4bff265cf897b5304f15f9f43579d6c/testFamily/c06cf4aadf634f1ea8f125bb08bd0240, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T22:59:48,656 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a4bff265cf897b5304f15f9f43579d6c in 447ms, sequenceid=8, compaction requested=false 2024-12-03T22:59:48,661 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testNormalScanTimeoutOnOpenScanner/a4bff265cf897b5304f15f9f43579d6c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T22:59:48,661 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. 2024-12-03T22:59:48,661 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a4bff265cf897b5304f15f9f43579d6c: Waiting for close lock at 1733266788209Running coprocessor pre-close hooks at 1733266788209Disabling compacts and flushes for region at 1733266788209Disabling writes for close at 1733266788209Obtaining lock to block concurrent updates at 1733266788209Preparing flush snapshotting stores in a4bff265cf897b5304f15f9f43579d6c at 1733266788209Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733266788209Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. at 1733266788210 (+1 ms)Flushing a4bff265cf897b5304f15f9f43579d6c/testFamily: creating writer at 1733266788210Flushing a4bff265cf897b5304f15f9f43579d6c/testFamily: appending metadata at 1733266788224 (+14 ms)Flushing a4bff265cf897b5304f15f9f43579d6c/testFamily: closing flushed file at 1733266788224Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4317a0d7: reopening flushed file at 1733266788645 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a4bff265cf897b5304f15f9f43579d6c in 447ms, sequenceid=8, compaction requested=false at 1733266788656 (+11 ms)Writing region close event to WAL at 1733266788657 (+1 ms)Running coprocessor post-close hooks at 1733266788661 (+4 ms)Closed at 1733266788661 2024-12-03T22:59:48,662 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733266745227.a4bff265cf897b5304f15f9f43579d6c. 2024-12-03T22:59:48,662 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d9c7e273746924274193a9471bc0898e, disabling compactions & flushes 2024-12-03T22:59:48,662 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. 2024-12-03T22:59:48,662 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. 2024-12-03T22:59:48,662 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. after waiting 0 ms 2024-12-03T22:59:48,662 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. 2024-12-03T22:59:48,662 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d9c7e273746924274193a9471bc0898e 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T22:59:48,677 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9c7e273746924274193a9471bc0898e/.tmp/testFamily/7f8ed5a4c3fe42f5a520efa2d6469272 is 61, key is row-0/testFamily:testQualifier/1733266766571/Put/seqid=0 2024-12-03T22:59:48,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741864_1040 (size=5049) 2024-12-03T22:59:48,691 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:48,892 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-12-03T22:59:48,892 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1325): Online Regions={d9c7e273746924274193a9471bc0898e=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e., 897bed5ed2ecae70c69cd2703f2323f9=hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9., 3cd9eb5ad6480573474415850e7c0740=testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740., 78246d6932357f4d3bdead6e1777c4c5=hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5.} 2024-12-03T22:59:48,892 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:48,930 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-03T22:59:48,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-03T22:59:48,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T22:59:48,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T22:59:48,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-03T22:59:48,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-03T22:59:48,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T22:59:49,084 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9c7e273746924274193a9471bc0898e/.tmp/testFamily/7f8ed5a4c3fe42f5a520efa2d6469272 2024-12-03T22:59:49,093 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9, d9c7e273746924274193a9471bc0898e 2024-12-03T22:59:49,102 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9c7e273746924274193a9471bc0898e/.tmp/testFamily/7f8ed5a4c3fe42f5a520efa2d6469272 as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9c7e273746924274193a9471bc0898e/testFamily/7f8ed5a4c3fe42f5a520efa2d6469272 2024-12-03T22:59:49,111 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9c7e273746924274193a9471bc0898e/testFamily/7f8ed5a4c3fe42f5a520efa2d6469272, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T22:59:49,113 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d9c7e273746924274193a9471bc0898e in 450ms, sequenceid=8, compaction requested=false 2024-12-03T22:59:49,118 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9c7e273746924274193a9471bc0898e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T22:59:49,119 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. 2024-12-03T22:59:49,119 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d9c7e273746924274193a9471bc0898e: Waiting for close lock at 1733266788662Running coprocessor pre-close hooks at 1733266788662Disabling compacts and flushes for region at 1733266788662Disabling writes for close at 1733266788662Obtaining lock to block concurrent updates at 1733266788662Preparing flush snapshotting stores in d9c7e273746924274193a9471bc0898e at 1733266788662Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733266788662Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. at 1733266788663 (+1 ms)Flushing d9c7e273746924274193a9471bc0898e/testFamily: creating writer at 1733266788663Flushing d9c7e273746924274193a9471bc0898e/testFamily: appending metadata at 1733266788677 (+14 ms)Flushing d9c7e273746924274193a9471bc0898e/testFamily: closing flushed file at 1733266788677Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@422f336b: reopening flushed file at 1733266789100 (+423 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d9c7e273746924274193a9471bc0898e in 450ms, sequenceid=8, compaction requested=false at 1733266789113 (+13 ms)Writing region close event to WAL at 1733266789114 (+1 ms)Running coprocessor post-close hooks at 1733266789119 (+5 ms)Closed at 1733266789119 2024-12-03T22:59:49,119 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733266763525.d9c7e273746924274193a9471bc0898e. 2024-12-03T22:59:49,120 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 897bed5ed2ecae70c69cd2703f2323f9, disabling compactions & flushes 2024-12-03T22:59:49,120 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. 2024-12-03T22:59:49,120 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. 2024-12-03T22:59:49,120 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. after waiting 0 ms 2024-12-03T22:59:49,120 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. 2024-12-03T22:59:49,120 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 897bed5ed2ecae70c69cd2703f2323f9 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T22:59:49,135 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScanner/897bed5ed2ecae70c69cd2703f2323f9/.tmp/testFamily/8cf04fb60b5d4e5fa738f124df56ae61 is 61, key is row-0/testFamily:testQualifier/1733266728985/Put/seqid=0 2024-12-03T22:59:49,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741865_1041 (size=5049) 2024-12-03T22:59:49,293 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9 2024-12-03T22:59:49,494 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5, 897bed5ed2ecae70c69cd2703f2323f9 2024-12-03T22:59:49,541 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScanner/897bed5ed2ecae70c69cd2703f2323f9/.tmp/testFamily/8cf04fb60b5d4e5fa738f124df56ae61 2024-12-03T22:59:49,556 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScanner/897bed5ed2ecae70c69cd2703f2323f9/.tmp/testFamily/8cf04fb60b5d4e5fa738f124df56ae61 as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScanner/897bed5ed2ecae70c69cd2703f2323f9/testFamily/8cf04fb60b5d4e5fa738f124df56ae61 2024-12-03T22:59:49,563 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScanner/897bed5ed2ecae70c69cd2703f2323f9/testFamily/8cf04fb60b5d4e5fa738f124df56ae61, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T22:59:49,564 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 897bed5ed2ecae70c69cd2703f2323f9 in 444ms, sequenceid=8, compaction requested=false 2024-12-03T22:59:49,570 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnOpenScanner/897bed5ed2ecae70c69cd2703f2323f9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T22:59:49,570 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. 2024-12-03T22:59:49,570 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 897bed5ed2ecae70c69cd2703f2323f9: Waiting for close lock at 1733266789119Running coprocessor pre-close hooks at 1733266789119Disabling compacts and flushes for region at 1733266789119Disabling writes for close at 1733266789120 (+1 ms)Obtaining lock to block concurrent updates at 1733266789120Preparing flush snapshotting stores in 897bed5ed2ecae70c69cd2703f2323f9 at 1733266789120Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733266789120Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. at 1733266789121 (+1 ms)Flushing 897bed5ed2ecae70c69cd2703f2323f9/testFamily: creating writer at 1733266789121Flushing 897bed5ed2ecae70c69cd2703f2323f9/testFamily: appending metadata at 1733266789135 (+14 ms)Flushing 897bed5ed2ecae70c69cd2703f2323f9/testFamily: closing flushed file at 1733266789135Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f57a69e: reopening flushed file at 1733266789555 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 897bed5ed2ecae70c69cd2703f2323f9 in 444ms, sequenceid=8, compaction requested=false at 1733266789565 (+10 ms)Writing region close event to WAL at 1733266789565Running coprocessor post-close hooks at 1733266789570 (+5 ms)Closed at 1733266789570 2024-12-03T22:59:49,570 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733266725948.897bed5ed2ecae70c69cd2703f2323f9. 2024-12-03T22:59:49,571 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3cd9eb5ad6480573474415850e7c0740, disabling compactions & flushes 2024-12-03T22:59:49,571 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. 2024-12-03T22:59:49,571 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. 2024-12-03T22:59:49,571 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. after waiting 0 ms 2024-12-03T22:59:49,571 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. 2024-12-03T22:59:49,571 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3cd9eb5ad6480573474415850e7c0740 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T22:59:49,586 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextException/3cd9eb5ad6480573474415850e7c0740/.tmp/testFamily/4b2e01c84dc046f5afb79044fc1e0609 is 61, key is row-0/testFamily:testQualifier/1733266745167/Put/seqid=0 2024-12-03T22:59:49,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741866_1042 (size=5049) 2024-12-03T22:59:49,694 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:59:49,894 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-03T22:59:49,895 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1325): Online Regions={3cd9eb5ad6480573474415850e7c0740=testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740., 78246d6932357f4d3bdead6e1777c4c5=hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5.} 2024-12-03T22:59:49,895 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 3cd9eb5ad6480573474415850e7c0740, 78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:59:49,993 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextException/3cd9eb5ad6480573474415850e7c0740/.tmp/testFamily/4b2e01c84dc046f5afb79044fc1e0609 2024-12-03T22:59:50,011 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextException/3cd9eb5ad6480573474415850e7c0740/.tmp/testFamily/4b2e01c84dc046f5afb79044fc1e0609 as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextException/3cd9eb5ad6480573474415850e7c0740/testFamily/4b2e01c84dc046f5afb79044fc1e0609 2024-12-03T22:59:50,019 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextException/3cd9eb5ad6480573474415850e7c0740/testFamily/4b2e01c84dc046f5afb79044fc1e0609, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T22:59:50,020 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3cd9eb5ad6480573474415850e7c0740 in 449ms, sequenceid=8, compaction requested=false 2024-12-03T22:59:50,024 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/default/testRetryOutOfOrderScannerNextException/3cd9eb5ad6480573474415850e7c0740/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T22:59:50,025 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. 2024-12-03T22:59:50,025 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3cd9eb5ad6480573474415850e7c0740: Waiting for close lock at 1733266789571Running coprocessor pre-close hooks at 1733266789571Disabling compacts and flushes for region at 1733266789571Disabling writes for close at 1733266789571Obtaining lock to block concurrent updates at 1733266789571Preparing flush snapshotting stores in 3cd9eb5ad6480573474415850e7c0740 at 1733266789571Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733266789571Flushing stores of testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. at 1733266789572 (+1 ms)Flushing 3cd9eb5ad6480573474415850e7c0740/testFamily: creating writer at 1733266789572Flushing 3cd9eb5ad6480573474415850e7c0740/testFamily: appending metadata at 1733266789586 (+14 ms)Flushing 3cd9eb5ad6480573474415850e7c0740/testFamily: closing flushed file at 1733266789586Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57fe6f52: reopening flushed file at 1733266790010 (+424 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3cd9eb5ad6480573474415850e7c0740 in 449ms, sequenceid=8, compaction requested=false at 1733266790020 (+10 ms)Writing region close event to WAL at 1733266790020Running coprocessor post-close hooks at 1733266790025 (+5 ms)Closed at 1733266790025 2024-12-03T22:59:50,025 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733266744126.3cd9eb5ad6480573474415850e7c0740. 2024-12-03T22:59:50,025 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 78246d6932357f4d3bdead6e1777c4c5, disabling compactions & flushes 2024-12-03T22:59:50,025 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. 2024-12-03T22:59:50,026 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. 2024-12-03T22:59:50,026 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. after waiting 0 ms 2024-12-03T22:59:50,026 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. 2024-12-03T22:59:50,026 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 78246d6932357f4d3bdead6e1777c4c5 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T22:59:50,039 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNextAsync/78246d6932357f4d3bdead6e1777c4c5/.tmp/testFamily/9279fd832b0543f2bde7b9b8baeee388 is 61, key is row-0/testFamily:testQualifier/1733266738086/Put/seqid=0 2024-12-03T22:59:50,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741867_1043 (size=5049) 2024-12-03T22:59:50,095 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:59:50,296 DEBUG [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1351): Waiting on 78246d6932357f4d3bdead6e1777c4c5 2024-12-03T22:59:50,446 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNextAsync/78246d6932357f4d3bdead6e1777c4c5/.tmp/testFamily/9279fd832b0543f2bde7b9b8baeee388 2024-12-03T22:59:50,462 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNextAsync/78246d6932357f4d3bdead6e1777c4c5/.tmp/testFamily/9279fd832b0543f2bde7b9b8baeee388 as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNextAsync/78246d6932357f4d3bdead6e1777c4c5/testFamily/9279fd832b0543f2bde7b9b8baeee388 2024-12-03T22:59:50,471 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNextAsync/78246d6932357f4d3bdead6e1777c4c5/testFamily/9279fd832b0543f2bde7b9b8baeee388, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T22:59:50,472 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 78246d6932357f4d3bdead6e1777c4c5 in 446ms, sequenceid=8, compaction requested=false 2024-12-03T22:59:50,476 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/data/hbase/testMetaScanTimeoutOnNextAsync/78246d6932357f4d3bdead6e1777c4c5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T22:59:50,477 INFO [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. 2024-12-03T22:59:50,477 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 78246d6932357f4d3bdead6e1777c4c5: Waiting for close lock at 1733266790025Running coprocessor pre-close hooks at 1733266790025Disabling compacts and flushes for region at 1733266790025Disabling writes for close at 1733266790026 (+1 ms)Obtaining lock to block concurrent updates at 1733266790026Preparing flush snapshotting stores in 78246d6932357f4d3bdead6e1777c4c5 at 1733266790026Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733266790026Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. at 1733266790027 (+1 ms)Flushing 78246d6932357f4d3bdead6e1777c4c5/testFamily: creating writer at 1733266790027Flushing 78246d6932357f4d3bdead6e1777c4c5/testFamily: appending metadata at 1733266790039 (+12 ms)Flushing 78246d6932357f4d3bdead6e1777c4c5/testFamily: closing flushed file at 1733266790039Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11d2a5a0: reopening flushed file at 1733266790461 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 78246d6932357f4d3bdead6e1777c4c5 in 446ms, sequenceid=8, compaction requested=false at 1733266790472 (+11 ms)Writing region close event to WAL at 1733266790473 (+1 ms)Running coprocessor post-close hooks at 1733266790477 (+4 ms)Closed at 1733266790477 2024-12-03T22:59:50,477 DEBUG [RS_CLOSE_REGION-regionserver/ed8014902436:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733266735043.78246d6932357f4d3bdead6e1777c4c5. 2024-12-03T22:59:50,496 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(976): stopping server ed8014902436,36731,1733266689308; all regions closed. 2024-12-03T22:59:50,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741834_1010 (size=19479) 2024-12-03T22:59:50,509 DEBUG [RS:0;ed8014902436:36731 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/oldWALs 2024-12-03T22:59:50,509 INFO [RS:0;ed8014902436:36731 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ed8014902436%2C36731%2C1733266689308.meta:.meta(num 1733266722344) 2024-12-03T22:59:50,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741833_1009 (size=20031) 2024-12-03T22:59:50,513 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/WALs/ed8014902436,36731,1733266689308/ed8014902436%2C36731%2C1733266689308.1733266721867 not finished, retry = 0 2024-12-03T22:59:50,621 DEBUG [RS:0;ed8014902436:36731 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/oldWALs 2024-12-03T22:59:50,621 INFO [RS:0;ed8014902436:36731 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ed8014902436%2C36731%2C1733266689308:(num 1733266721867) 2024-12-03T22:59:50,621 DEBUG [RS:0;ed8014902436:36731 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T22:59:50,621 INFO [RS:0;ed8014902436:36731 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T22:59:50,622 INFO [RS:0;ed8014902436:36731 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T22:59:50,623 INFO [RS:0;ed8014902436:36731 {}] hbase.ChoreService(370): Chore service for: regionserver/ed8014902436:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T22:59:50,623 INFO [RS:0;ed8014902436:36731 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T22:59:50,623 INFO [regionserver/ed8014902436:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T22:59:50,626 INFO [RS:0;ed8014902436:36731 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36731 2024-12-03T22:59:50,630 INFO [regionserver/ed8014902436:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T22:59:50,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ed8014902436,36731,1733266689308 2024-12-03T22:59:50,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T22:59:50,677 INFO [RS:0;ed8014902436:36731 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T22:59:50,679 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ed8014902436,36731,1733266689308] 2024-12-03T22:59:50,701 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ed8014902436,36731,1733266689308 already deleted, retry=false 2024-12-03T22:59:50,702 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ed8014902436,36731,1733266689308 expired; onlineServers=0 2024-12-03T22:59:50,702 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ed8014902436,44997,1733266688559' ***** 2024-12-03T22:59:50,702 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T22:59:50,703 INFO [M:0;ed8014902436:44997 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T22:59:50,703 INFO [M:0;ed8014902436:44997 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T22:59:50,703 DEBUG [M:0;ed8014902436:44997 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T22:59:50,704 DEBUG [M:0;ed8014902436:44997 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T22:59:50,704 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T22:59:50,704 DEBUG [master/ed8014902436:0:becomeActiveMaster-HFileCleaner.small.0-1733266691626 {}] cleaner.HFileCleaner(306): Exit Thread[master/ed8014902436:0:becomeActiveMaster-HFileCleaner.small.0-1733266691626,5,FailOnTimeoutGroup] 2024-12-03T22:59:50,704 DEBUG [master/ed8014902436:0:becomeActiveMaster-HFileCleaner.large.0-1733266691626 {}] cleaner.HFileCleaner(306): Exit Thread[master/ed8014902436:0:becomeActiveMaster-HFileCleaner.large.0-1733266691626,5,FailOnTimeoutGroup] 2024-12-03T22:59:50,705 INFO [M:0;ed8014902436:44997 {}] hbase.ChoreService(370): Chore service for: master/ed8014902436:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T22:59:50,705 INFO [M:0;ed8014902436:44997 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T22:59:50,705 DEBUG [M:0;ed8014902436:44997 {}] master.HMaster(1795): Stopping service threads 2024-12-03T22:59:50,706 INFO [M:0;ed8014902436:44997 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T22:59:50,706 INFO [M:0;ed8014902436:44997 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T22:59:50,707 INFO [M:0;ed8014902436:44997 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T22:59:50,707 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T22:59:50,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T22:59:50,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T22:59:50,715 DEBUG [M:0;ed8014902436:44997 {}] zookeeper.ZKUtil(347): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T22:59:50,716 WARN [M:0;ed8014902436:44997 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T22:59:50,717 INFO [M:0;ed8014902436:44997 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/.lastflushedseqids 2024-12-03T22:59:50,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741868_1044 (size=649) 2024-12-03T22:59:50,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T22:59:50,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36731-0x1019eb459510001, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T22:59:50,792 INFO [RS:0;ed8014902436:36731 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T22:59:50,793 INFO [RS:0;ed8014902436:36731 {}] regionserver.HRegionServer(1031): Exiting; stopping=ed8014902436,36731,1733266689308; zookeeper connection closed. 2024-12-03T22:59:50,793 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@10578220 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@10578220 2024-12-03T22:59:50,794 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-03T22:59:51,132 INFO [M:0;ed8014902436:44997 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T22:59:51,133 INFO [M:0;ed8014902436:44997 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T22:59:51,133 DEBUG [M:0;ed8014902436:44997 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T22:59:51,133 INFO [M:0;ed8014902436:44997 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T22:59:51,133 DEBUG [M:0;ed8014902436:44997 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T22:59:51,134 DEBUG [M:0;ed8014902436:44997 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T22:59:51,134 DEBUG [M:0;ed8014902436:44997 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T22:59:51,134 INFO [M:0;ed8014902436:44997 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.00 KB heapSize=190.91 KB 2024-12-03T22:59:51,174 DEBUG [M:0;ed8014902436:44997 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c18ec21b8724b96808d42f674d1a3b9 is 82, key is hbase:meta,,1/info:regioninfo/1733266722431/Put/seqid=0 2024-12-03T22:59:51,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741869_1045 (size=5672) 2024-12-03T22:59:51,580 INFO [M:0;ed8014902436:44997 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c18ec21b8724b96808d42f674d1a3b9 2024-12-03T22:59:51,605 DEBUG [M:0;ed8014902436:44997 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e4656bc411c24c7cacb8bf90c7dc29f5 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733266724027/Put/seqid=0 2024-12-03T22:59:51,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741870_1046 (size=14070) 2024-12-03T22:59:52,012 INFO [M:0;ed8014902436:44997 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.45 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e4656bc411c24c7cacb8bf90c7dc29f5 2024-12-03T22:59:52,024 INFO [M:0;ed8014902436:44997 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e4656bc411c24c7cacb8bf90c7dc29f5 2024-12-03T22:59:52,042 DEBUG [M:0;ed8014902436:44997 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ac76c605e68432e857907423fe159a8 is 69, key is ed8014902436,36731,1733266689308/rs:state/1733266691732/Put/seqid=0 2024-12-03T22:59:52,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741871_1047 (size=5156) 2024-12-03T22:59:52,449 INFO [M:0;ed8014902436:44997 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ac76c605e68432e857907423fe159a8 2024-12-03T22:59:52,462 DEBUG [M:0;ed8014902436:44997 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c18ec21b8724b96808d42f674d1a3b9 as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2c18ec21b8724b96808d42f674d1a3b9 2024-12-03T22:59:52,469 INFO [M:0;ed8014902436:44997 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2c18ec21b8724b96808d42f674d1a3b9, entries=8, sequenceid=317, filesize=5.5 K 2024-12-03T22:59:52,471 DEBUG [M:0;ed8014902436:44997 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e4656bc411c24c7cacb8bf90c7dc29f5 as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e4656bc411c24c7cacb8bf90c7dc29f5 2024-12-03T22:59:52,477 INFO [M:0;ed8014902436:44997 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e4656bc411c24c7cacb8bf90c7dc29f5 2024-12-03T22:59:52,477 INFO [M:0;ed8014902436:44997 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e4656bc411c24c7cacb8bf90c7dc29f5, entries=33, sequenceid=317, filesize=13.7 K 2024-12-03T22:59:52,478 DEBUG [M:0;ed8014902436:44997 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ac76c605e68432e857907423fe159a8 as hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5ac76c605e68432e857907423fe159a8 2024-12-03T22:59:52,485 INFO [M:0;ed8014902436:44997 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41127/user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5ac76c605e68432e857907423fe159a8, entries=1, sequenceid=317, filesize=5.0 K 2024-12-03T22:59:52,486 INFO [M:0;ed8014902436:44997 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.00 KB/166915, heapSize ~190.61 KB/195184, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1352ms, sequenceid=317, compaction requested=false 2024-12-03T22:59:52,491 INFO [M:0;ed8014902436:44997 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T22:59:52,491 DEBUG [M:0;ed8014902436:44997 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733266791133Disabling compacts and flushes for region at 1733266791133Disabling writes for close at 1733266791134 (+1 ms)Obtaining lock to block concurrent updates at 1733266791134Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733266791134Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166915, getHeapSize=195424, getOffHeapSize=0, getCellsCount=364 at 1733266791135 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733266791151 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733266791151Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733266791174 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733266791174Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733266791588 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733266791604 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733266791604Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733266792025 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733266792041 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733266792041Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42b36faf: reopening flushed file at 1733266792461 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52ac1951: reopening flushed file at 1733266792470 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72723215: reopening flushed file at 1733266792477 (+7 ms)Finished flush of dataSize ~163.00 KB/166915, heapSize ~190.61 KB/195184, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1352ms, sequenceid=317, compaction requested=false at 1733266792486 (+9 ms)Writing region close event to WAL at 1733266792491 (+5 ms)Closed at 1733266792491 2024-12-03T22:59:52,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36673 is added to blk_1073741830_1006 (size=188127) 2024-12-03T22:59:52,494 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/9cc9174c-1459-f4f8-94c8-07f0bea17e58/MasterData/WALs/ed8014902436,44997,1733266688559/ed8014902436%2C44997%2C1733266688559.1733266691007 not finished, retry = 0 2024-12-03T22:59:52,595 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T22:59:52,595 INFO [M:0;ed8014902436:44997 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T22:59:52,596 INFO [M:0;ed8014902436:44997 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44997 2024-12-03T22:59:52,596 INFO [M:0;ed8014902436:44997 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T22:59:52,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T22:59:52,765 INFO [M:0;ed8014902436:44997 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T22:59:52,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44997-0x1019eb459510000, quorum=127.0.0.1:55134, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T22:59:52,770 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T22:59:52,771 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T22:59:52,771 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T22:59:52,771 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T22:59:52,772 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/hadoop.log.dir/,STOPPED} 2024-12-03T22:59:52,773 WARN [BP-1108234711-172.17.0.2-1733266685190 heartbeating to localhost/127.0.0.1:41127 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T22:59:52,773 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T22:59:52,773 WARN [BP-1108234711-172.17.0.2-1733266685190 heartbeating to localhost/127.0.0.1:41127 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1108234711-172.17.0.2-1733266685190 (Datanode Uuid ee7209ab-e464-43bb-9153-701373c951ef) service to localhost/127.0.0.1:41127 2024-12-03T22:59:52,774 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T22:59:52,775 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/cluster_a42b0e19-4ea0-5ed7-aabb-16b1ec3e8836/data/data1/current/BP-1108234711-172.17.0.2-1733266685190 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T22:59:52,775 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/cluster_a42b0e19-4ea0-5ed7-aabb-16b1ec3e8836/data/data2/current/BP-1108234711-172.17.0.2-1733266685190 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T22:59:52,776 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T22:59:52,784 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T22:59:52,785 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T22:59:52,785 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T22:59:52,785 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T22:59:52,785 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d052811d-8daf-d347-d581-998e011ccc18/hadoop.log.dir/,STOPPED} 2024-12-03T22:59:52,793 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T22:59:52,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down