2024-12-03 14:57:20,052 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-03 14:57:20,063 main DEBUG Took 0.009195 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 14:57:20,063 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 14:57:20,063 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 14:57:20,070 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 14:57:20,071 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:57:20,077 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 14:57:20,088 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:57:20,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:57:20,090 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:57:20,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:57:20,090 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:57:20,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:57:20,091 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:57:20,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:57:20,092 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:57:20,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:57:20,093 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:57:20,093 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:57:20,094 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:57:20,094 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:57:20,094 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:57:20,094 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:57:20,095 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:57:20,095 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:57:20,096 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:57:20,096 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:57:20,096 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:57:20,096 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:57:20,097 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:57:20,097 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:57:20,097 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:57:20,098 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 14:57:20,099 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:57:20,100 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 14:57:20,102 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 14:57:20,102 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 14:57:20,103 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 14:57:20,103 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 14:57:20,111 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 14:57:20,113 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 14:57:20,115 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 14:57:20,115 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 14:57:20,115 main DEBUG createAppenders(={Console}) 2024-12-03 14:57:20,116 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-03 14:57:20,117 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-03 14:57:20,117 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-03 14:57:20,117 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 14:57:20,118 main DEBUG OutputStream closed 2024-12-03 14:57:20,118 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 14:57:20,118 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 14:57:20,118 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-03 14:57:20,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 14:57:20,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 14:57:20,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 14:57:20,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 14:57:20,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 14:57:20,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 14:57:20,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 14:57:20,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 14:57:20,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 14:57:20,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 14:57:20,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 14:57:20,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 14:57:20,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 14:57:20,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 14:57:20,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 14:57:20,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 14:57:20,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 14:57:20,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 14:57:20,194 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 14:57:20,194 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-03 14:57:20,194 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 14:57:20,195 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-03T14:57:20,207 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-03 14:57:20,210 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 14:57:20,210 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T14:57:20,398 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a 2024-12-03T14:57:20,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T14:57:20,435 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/cluster_e6ebabe5-9f0b-8d92-13b6-dc963c44e6c1, deleteOnExit=true 2024-12-03T14:57:20,435 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T14:57:20,436 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/test.cache.data in system properties and HBase conf 2024-12-03T14:57:20,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T14:57:20,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/hadoop.log.dir in system properties and HBase conf 2024-12-03T14:57:20,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T14:57:20,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T14:57:20,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T14:57:20,549 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T14:57:20,645 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T14:57:20,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T14:57:20,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T14:57:20,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T14:57:20,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T14:57:20,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T14:57:20,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T14:57:20,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T14:57:20,652 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T14:57:20,652 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T14:57:20,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/nfs.dump.dir in system properties and HBase conf 2024-12-03T14:57:20,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/java.io.tmpdir in system properties and HBase conf 2024-12-03T14:57:20,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T14:57:20,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T14:57:20,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T14:57:21,342 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T14:57:21,410 INFO [Time-limited test {}] log.Log(170): Logging initialized @2006ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T14:57:21,473 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T14:57:21,527 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T14:57:21,545 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T14:57:21,546 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T14:57:21,547 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T14:57:21,558 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T14:57:21,560 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@660c56f7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/hadoop.log.dir/,AVAILABLE} 2024-12-03T14:57:21,561 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45f63f41{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T14:57:21,743 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@59bae7b3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/java.io.tmpdir/jetty-localhost-35029-hadoop-hdfs-3_4_1-tests_jar-_-any-13422449011341589592/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T14:57:21,751 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f237650{HTTP/1.1, (http/1.1)}{localhost:35029} 2024-12-03T14:57:21,751 INFO [Time-limited test {}] server.Server(415): Started @2349ms 2024-12-03T14:57:22,046 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T14:57:22,054 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T14:57:22,055 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T14:57:22,055 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T14:57:22,055 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T14:57:22,056 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d304465{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/hadoop.log.dir/,AVAILABLE} 2024-12-03T14:57:22,057 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c1ee62c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T14:57:22,162 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@258822a0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/java.io.tmpdir/jetty-localhost-40037-hadoop-hdfs-3_4_1-tests_jar-_-any-4744789397370026452/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T14:57:22,163 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@514e86a2{HTTP/1.1, (http/1.1)}{localhost:40037} 2024-12-03T14:57:22,163 INFO [Time-limited test {}] server.Server(415): Started @2761ms 2024-12-03T14:57:22,214 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T14:57:22,662 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/cluster_e6ebabe5-9f0b-8d92-13b6-dc963c44e6c1/data/data1/current/BP-359429629-172.17.0.2-1733237841157/current, will proceed with Du for space computation calculation, 2024-12-03T14:57:22,662 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/cluster_e6ebabe5-9f0b-8d92-13b6-dc963c44e6c1/data/data2/current/BP-359429629-172.17.0.2-1733237841157/current, will proceed with Du for space computation calculation, 2024-12-03T14:57:22,693 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T14:57:22,743 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1712fb50422a6939 with lease ID 0x2c375ff765a6f7cb: Processing first storage report for DS-7c91f156-a9ca-41a5-83a0-60bb0cfbd6c2 from datanode DatanodeRegistration(127.0.0.1:44763, datanodeUuid=b10844da-bc93-4a91-a401-e2f18c7da2ef, infoPort=32915, infoSecurePort=0, ipcPort=34707, storageInfo=lv=-57;cid=testClusterID;nsid=1909314181;c=1733237841157) 2024-12-03T14:57:22,744 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1712fb50422a6939 with lease ID 0x2c375ff765a6f7cb: from storage DS-7c91f156-a9ca-41a5-83a0-60bb0cfbd6c2 node DatanodeRegistration(127.0.0.1:44763, datanodeUuid=b10844da-bc93-4a91-a401-e2f18c7da2ef, infoPort=32915, infoSecurePort=0, ipcPort=34707, storageInfo=lv=-57;cid=testClusterID;nsid=1909314181;c=1733237841157), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T14:57:22,745 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1712fb50422a6939 with lease ID 0x2c375ff765a6f7cb: Processing first storage report for DS-1e4c171b-b91e-4193-a01f-ce58b28ee6e2 from datanode DatanodeRegistration(127.0.0.1:44763, datanodeUuid=b10844da-bc93-4a91-a401-e2f18c7da2ef, infoPort=32915, infoSecurePort=0, ipcPort=34707, storageInfo=lv=-57;cid=testClusterID;nsid=1909314181;c=1733237841157) 2024-12-03T14:57:22,745 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1712fb50422a6939 with lease ID 0x2c375ff765a6f7cb: from storage DS-1e4c171b-b91e-4193-a01f-ce58b28ee6e2 node DatanodeRegistration(127.0.0.1:44763, datanodeUuid=b10844da-bc93-4a91-a401-e2f18c7da2ef, infoPort=32915, infoSecurePort=0, ipcPort=34707, storageInfo=lv=-57;cid=testClusterID;nsid=1909314181;c=1733237841157), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T14:57:22,803 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a 2024-12-03T14:57:22,868 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/cluster_e6ebabe5-9f0b-8d92-13b6-dc963c44e6c1/zookeeper_0, clientPort=49750, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/cluster_e6ebabe5-9f0b-8d92-13b6-dc963c44e6c1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/cluster_e6ebabe5-9f0b-8d92-13b6-dc963c44e6c1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T14:57:22,877 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49750 2024-12-03T14:57:22,891 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:57:22,896 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:57:23,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741825_1001 (size=7) 2024-12-03T14:57:23,580 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e with version=8 2024-12-03T14:57:23,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/hbase-staging 2024-12-03T14:57:23,652 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T14:57:23,848 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a5d22df9eca2:0 server-side Connection retries=9 2024-12-03T14:57:23,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:57:23,858 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T14:57:23,862 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T14:57:23,862 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:57:23,863 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T14:57:23,991 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T14:57:24,045 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T14:57:24,053 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T14:57:24,057 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T14:57:24,079 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 125290 (auto-detected) 2024-12-03T14:57:24,080 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-03T14:57:24,097 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44239 2024-12-03T14:57:24,120 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44239 connecting to ZooKeeper ensemble=127.0.0.1:49750 2024-12-03T14:57:24,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:442390x0, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T14:57:24,148 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44239-0x100a09600120000 connected 2024-12-03T14:57:24,176 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:57:24,179 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:57:24,189 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:57:24,192 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e, hbase.cluster.distributed=false 2024-12-03T14:57:24,216 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T14:57:24,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44239 2024-12-03T14:57:24,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44239 2024-12-03T14:57:24,221 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44239 2024-12-03T14:57:24,221 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44239 2024-12-03T14:57:24,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44239 2024-12-03T14:57:24,318 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a5d22df9eca2:0 server-side Connection retries=9 2024-12-03T14:57:24,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:57:24,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T14:57:24,320 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T14:57:24,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:57:24,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T14:57:24,323 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T14:57:24,325 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T14:57:24,325 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45227 2024-12-03T14:57:24,327 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45227 connecting to ZooKeeper ensemble=127.0.0.1:49750 2024-12-03T14:57:24,328 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:57:24,331 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:57:24,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452270x0, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T14:57:24,336 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:452270x0, quorum=127.0.0.1:49750, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:57:24,337 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45227-0x100a09600120001 connected 2024-12-03T14:57:24,340 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T14:57:24,349 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T14:57:24,352 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T14:57:24,356 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T14:57:24,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45227 2024-12-03T14:57:24,358 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45227 2024-12-03T14:57:24,358 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45227 2024-12-03T14:57:24,359 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45227 2024-12-03T14:57:24,359 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45227 2024-12-03T14:57:24,373 DEBUG [M:0;a5d22df9eca2:44239 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a5d22df9eca2:44239 2024-12-03T14:57:24,374 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a5d22df9eca2,44239,1733237843695 2024-12-03T14:57:24,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:57:24,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:57:24,381 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a5d22df9eca2,44239,1733237843695 2024-12-03T14:57:24,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T14:57:24,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:57:24,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:57:24,399 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T14:57:24,400 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a5d22df9eca2,44239,1733237843695 from backup master directory 2024-12-03T14:57:24,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a5d22df9eca2,44239,1733237843695 2024-12-03T14:57:24,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:57:24,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:57:24,403 WARN [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T14:57:24,403 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a5d22df9eca2,44239,1733237843695 2024-12-03T14:57:24,405 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T14:57:24,406 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T14:57:24,455 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/hbase.id] with ID: 1d8f4de9-4ea3-498e-9402-5f1d161c99d2 2024-12-03T14:57:24,455 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/.tmp/hbase.id 2024-12-03T14:57:24,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741826_1002 (size=42) 2024-12-03T14:57:24,870 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/.tmp/hbase.id]:[hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/hbase.id] 2024-12-03T14:57:24,920 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:57:24,924 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T14:57:24,940 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-03T14:57:24,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:57:24,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:57:24,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741827_1003 (size=196) 2024-12-03T14:57:25,376 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:57:25,378 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T14:57:25,392 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:57:25,395 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:57:25,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741828_1004 (size=1189) 2024-12-03T14:57:25,847 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store 2024-12-03T14:57:25,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741829_1005 (size=34) 2024-12-03T14:57:26,274 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T14:57:26,279 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:57:26,280 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T14:57:26,280 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:57:26,280 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:57:26,281 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T14:57:26,282 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:57:26,282 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:57:26,283 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733237846280Disabling compacts and flushes for region at 1733237846280Disabling writes for close at 1733237846281 (+1 ms)Writing region close event to WAL at 1733237846282 (+1 ms)Closed at 1733237846282 2024-12-03T14:57:26,285 WARN [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/.initializing 2024-12-03T14:57:26,285 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/WALs/a5d22df9eca2,44239,1733237843695 2024-12-03T14:57:26,293 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:57:26,306 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C44239%2C1733237843695, suffix=, logDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/WALs/a5d22df9eca2,44239,1733237843695, archiveDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/oldWALs, maxLogs=10 2024-12-03T14:57:26,326 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/WALs/a5d22df9eca2,44239,1733237843695/a5d22df9eca2%2C44239%2C1733237843695.1733237846310, exclude list is [], retry=0 2024-12-03T14:57:26,342 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44763,DS-7c91f156-a9ca-41a5-83a0-60bb0cfbd6c2,DISK] 2024-12-03T14:57:26,345 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T14:57:26,377 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/WALs/a5d22df9eca2,44239,1733237843695/a5d22df9eca2%2C44239%2C1733237843695.1733237846310 2024-12-03T14:57:26,378 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32915:32915)] 2024-12-03T14:57:26,378 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:57:26,378 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:57:26,381 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:57:26,382 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:57:26,415 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:57:26,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T14:57:26,444 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:26,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:57:26,448 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:57:26,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T14:57:26,452 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:26,453 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:57:26,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:57:26,457 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T14:57:26,457 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:26,458 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:57:26,458 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:57:26,462 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T14:57:26,462 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:26,464 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:57:26,465 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:57:26,469 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:57:26,471 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:57:26,477 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:57:26,478 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:57:26,483 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T14:57:26,487 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:57:26,493 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:57:26,495 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61493611, jitterRate=-0.08367379009723663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T14:57:26,500 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733237846393Initializing all the Stores at 1733237846395 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237846396 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237846397 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237846397Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237846397Cleaning up temporary data from old regions at 1733237846479 (+82 ms)Region opened successfully at 1733237846500 (+21 ms) 2024-12-03T14:57:26,501 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T14:57:26,532 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57cf917b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0 2024-12-03T14:57:26,558 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T14:57:26,567 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T14:57:26,567 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T14:57:26,571 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T14:57:26,572 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T14:57:26,577 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-03T14:57:26,577 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T14:57:26,604 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T14:57:26,614 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T14:57:26,615 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T14:57:26,618 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T14:57:26,619 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T14:57:26,620 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T14:57:26,622 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T14:57:26,625 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T14:57:26,626 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T14:57:26,628 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T14:57:26,629 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T14:57:26,645 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T14:57:26,646 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T14:57:26,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T14:57:26,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T14:57:26,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:57:26,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:57:26,651 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a5d22df9eca2,44239,1733237843695, sessionid=0x100a09600120000, setting cluster-up flag (Was=false) 2024-12-03T14:57:26,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:57:26,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:57:26,664 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T14:57:26,666 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,44239,1733237843695 2024-12-03T14:57:26,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:57:26,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:57:26,676 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T14:57:26,677 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,44239,1733237843695 2024-12-03T14:57:26,683 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T14:57:26,751 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T14:57:26,759 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T14:57:26,766 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T14:57:26,766 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(746): ClusterId : 1d8f4de9-4ea3-498e-9402-5f1d161c99d2 2024-12-03T14:57:26,768 DEBUG [RS:0;a5d22df9eca2:45227 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T14:57:26,772 DEBUG [RS:0;a5d22df9eca2:45227 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T14:57:26,772 DEBUG [RS:0;a5d22df9eca2:45227 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T14:57:26,770 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a5d22df9eca2,44239,1733237843695 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T14:57:26,775 DEBUG [RS:0;a5d22df9eca2:45227 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T14:57:26,776 DEBUG [RS:0;a5d22df9eca2:45227 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204c075, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0 2024-12-03T14:57:26,777 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5 2024-12-03T14:57:26,777 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5 2024-12-03T14:57:26,777 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5 2024-12-03T14:57:26,777 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5 2024-12-03T14:57:26,777 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a5d22df9eca2:0, corePoolSize=10, maxPoolSize=10 2024-12-03T14:57:26,778 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:57:26,778 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2 2024-12-03T14:57:26,778 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:57:26,780 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733237876779 2024-12-03T14:57:26,781 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T14:57:26,782 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T14:57:26,784 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T14:57:26,784 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T14:57:26,785 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T14:57:26,786 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T14:57:26,786 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T14:57:26,786 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T14:57:26,787 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:26,790 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T14:57:26,791 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:26,792 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a5d22df9eca2:45227 2024-12-03T14:57:26,792 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T14:57:26,792 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T14:57:26,792 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T14:57:26,794 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T14:57:26,794 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T14:57:26,795 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T14:57:26,795 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T14:57:26,795 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T14:57:26,796 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237846796,5,FailOnTimeoutGroup] 2024-12-03T14:57:26,797 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237846796,5,FailOnTimeoutGroup] 2024-12-03T14:57:26,797 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:26,797 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T14:57:26,798 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(2659): reportForDuty to master=a5d22df9eca2,44239,1733237843695 with port=45227, startcode=1733237844286 2024-12-03T14:57:26,799 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:26,799 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:26,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741831_1007 (size=1321) 2024-12-03T14:57:26,808 DEBUG [RS:0;a5d22df9eca2:45227 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T14:57:26,835 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41965, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T14:57:26,841 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44239 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a5d22df9eca2,45227,1733237844286 2024-12-03T14:57:26,843 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44239 {}] master.ServerManager(517): Registering regionserver=a5d22df9eca2,45227,1733237844286 2024-12-03T14:57:26,855 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e 2024-12-03T14:57:26,855 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39033 2024-12-03T14:57:26,855 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T14:57:26,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T14:57:26,859 DEBUG [RS:0;a5d22df9eca2:45227 {}] zookeeper.ZKUtil(111): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a5d22df9eca2,45227,1733237844286 2024-12-03T14:57:26,860 WARN [RS:0;a5d22df9eca2:45227 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T14:57:26,860 INFO [RS:0;a5d22df9eca2:45227 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:57:26,860 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/WALs/a5d22df9eca2,45227,1733237844286 2024-12-03T14:57:26,862 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a5d22df9eca2,45227,1733237844286] 2024-12-03T14:57:26,883 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T14:57:26,895 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T14:57:26,900 INFO [RS:0;a5d22df9eca2:45227 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T14:57:26,900 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:26,901 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T14:57:26,906 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T14:57:26,908 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:26,908 DEBUG [RS:0;a5d22df9eca2:45227 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:57:26,908 DEBUG [RS:0;a5d22df9eca2:45227 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:57:26,908 DEBUG [RS:0;a5d22df9eca2:45227 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:57:26,909 DEBUG [RS:0;a5d22df9eca2:45227 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:57:26,909 DEBUG [RS:0;a5d22df9eca2:45227 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:57:26,909 DEBUG [RS:0;a5d22df9eca2:45227 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2 2024-12-03T14:57:26,909 DEBUG [RS:0;a5d22df9eca2:45227 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:57:26,909 DEBUG [RS:0;a5d22df9eca2:45227 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:57:26,909 DEBUG [RS:0;a5d22df9eca2:45227 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:57:26,909 DEBUG [RS:0;a5d22df9eca2:45227 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:57:26,910 DEBUG [RS:0;a5d22df9eca2:45227 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:57:26,910 DEBUG [RS:0;a5d22df9eca2:45227 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:57:26,910 DEBUG [RS:0;a5d22df9eca2:45227 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3 2024-12-03T14:57:26,910 DEBUG [RS:0;a5d22df9eca2:45227 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3 2024-12-03T14:57:26,911 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:26,911 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:26,911 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:26,911 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:26,911 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:26,912 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,45227,1733237844286-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T14:57:26,927 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T14:57:26,929 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,45227,1733237844286-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:26,929 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:26,929 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.Replication(171): a5d22df9eca2,45227,1733237844286 started 2024-12-03T14:57:26,952 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:26,953 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1482): Serving as a5d22df9eca2,45227,1733237844286, RpcServer on a5d22df9eca2/172.17.0.2:45227, sessionid=0x100a09600120001 2024-12-03T14:57:26,953 DEBUG [RS:0;a5d22df9eca2:45227 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T14:57:26,954 DEBUG [RS:0;a5d22df9eca2:45227 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a5d22df9eca2,45227,1733237844286 2024-12-03T14:57:26,954 DEBUG [RS:0;a5d22df9eca2:45227 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,45227,1733237844286' 2024-12-03T14:57:26,954 DEBUG [RS:0;a5d22df9eca2:45227 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T14:57:26,955 DEBUG [RS:0;a5d22df9eca2:45227 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T14:57:26,956 DEBUG [RS:0;a5d22df9eca2:45227 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T14:57:26,957 DEBUG [RS:0;a5d22df9eca2:45227 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T14:57:26,957 DEBUG [RS:0;a5d22df9eca2:45227 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a5d22df9eca2,45227,1733237844286 2024-12-03T14:57:26,957 DEBUG [RS:0;a5d22df9eca2:45227 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,45227,1733237844286' 2024-12-03T14:57:26,957 DEBUG [RS:0;a5d22df9eca2:45227 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T14:57:26,958 DEBUG [RS:0;a5d22df9eca2:45227 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T14:57:26,959 DEBUG [RS:0;a5d22df9eca2:45227 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T14:57:26,959 INFO [RS:0;a5d22df9eca2:45227 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T14:57:26,959 INFO [RS:0;a5d22df9eca2:45227 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T14:57:27,203 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T14:57:27,204 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e 2024-12-03T14:57:27,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741832_1008 (size=32) 2024-12-03T14:57:27,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:57:27,619 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T14:57:27,622 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T14:57:27,622 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:27,623 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:57:27,623 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T14:57:27,625 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T14:57:27,626 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:27,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:57:27,627 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T14:57:27,629 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T14:57:27,630 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:27,631 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:57:27,631 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T14:57:27,635 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T14:57:27,636 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:27,637 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:57:27,637 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T14:57:27,639 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740 2024-12-03T14:57:27,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740 2024-12-03T14:57:27,644 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T14:57:27,644 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T14:57:27,645 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T14:57:27,648 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T14:57:27,653 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:57:27,654 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70841566, jitterRate=0.055621594190597534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T14:57:27,657 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733237847615Initializing all the Stores at 1733237847618 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237847618Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237847618Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237847618Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237847618Cleaning up temporary data from old regions at 1733237847644 (+26 ms)Region opened successfully at 1733237847657 (+13 ms) 2024-12-03T14:57:27,657 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T14:57:27,658 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T14:57:27,658 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T14:57:27,658 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T14:57:27,659 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T14:57:27,660 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T14:57:27,661 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733237847657Disabling compacts and flushes for region at 1733237847657Disabling writes for close at 1733237847659 (+2 ms)Writing region close event to WAL at 1733237847660 (+1 ms)Closed at 1733237847660 2024-12-03T14:57:27,665 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T14:57:27,665 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T14:57:27,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T14:57:27,683 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T14:57:27,686 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T14:57:27,838 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:28,838 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:29,839 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:30,840 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:31,840 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:32,841 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:33,842 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:34,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T14:57:34,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T14:57:34,842 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:35,843 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:36,844 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:37,844 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:38,845 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:39,845 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:40,846 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:41,847 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:42,847 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:43,848 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:44,849 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:45,849 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:46,850 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:47,851 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:48,852 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:49,853 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:50,854 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:51,855 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:52,798 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T14:57:52,856 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:53,857 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:54,858 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:55,859 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:56,860 WARN [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:57:56,970 INFO [RS:0;a5d22df9eca2:45227 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:57:56,974 INFO [RS:0;a5d22df9eca2:45227 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C45227%2C1733237844286, suffix=, logDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/WALs/a5d22df9eca2,45227,1733237844286, archiveDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/oldWALs, maxLogs=32 2024-12-03T14:57:56,987 DEBUG [RS:0;a5d22df9eca2:45227 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/WALs/a5d22df9eca2,45227,1733237844286/a5d22df9eca2%2C45227%2C1733237844286.1733237876976, exclude list is [], retry=0 2024-12-03T14:57:56,991 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44763,DS-7c91f156-a9ca-41a5-83a0-60bb0cfbd6c2,DISK] 2024-12-03T14:57:56,993 INFO [RS:0;a5d22df9eca2:45227 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/WALs/a5d22df9eca2,45227,1733237844286/a5d22df9eca2%2C45227%2C1733237844286.1733237876976 2024-12-03T14:57:56,994 DEBUG [RS:0;a5d22df9eca2:45227 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32915:32915)] 2024-12-03T14:57:57,114 DEBUG [a5d22df9eca2:44239 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T14:57:57,126 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:57:57,131 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,45227,1733237844286, state=OPENING 2024-12-03T14:57:57,135 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T14:57:57,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:57:57,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:57:57,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:57:57,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:57:57,138 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T14:57:57,140 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,45227,1733237844286}] 2024-12-03T14:57:57,321 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T14:57:57,324 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33237, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T14:57:57,336 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T14:57:57,336 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:57:57,336 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T14:57:57,339 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C45227%2C1733237844286.meta, suffix=.meta, logDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/WALs/a5d22df9eca2,45227,1733237844286, archiveDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/oldWALs, maxLogs=32 2024-12-03T14:57:57,352 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/WALs/a5d22df9eca2,45227,1733237844286/a5d22df9eca2%2C45227%2C1733237844286.meta.1733237877341.meta, exclude list is [], retry=0 2024-12-03T14:57:57,355 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44763,DS-7c91f156-a9ca-41a5-83a0-60bb0cfbd6c2,DISK] 2024-12-03T14:57:57,358 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/WALs/a5d22df9eca2,45227,1733237844286/a5d22df9eca2%2C45227%2C1733237844286.meta.1733237877341.meta 2024-12-03T14:57:57,359 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32915:32915)] 2024-12-03T14:57:57,359 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:57:57,361 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T14:57:57,363 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T14:57:57,367 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T14:57:57,371 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T14:57:57,372 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:57:57,372 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T14:57:57,372 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T14:57:57,375 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T14:57:57,376 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T14:57:57,377 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:57,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:57:57,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T14:57:57,379 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T14:57:57,379 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:57,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:57:57,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T14:57:57,381 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T14:57:57,381 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:57,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:57:57,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T14:57:57,383 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T14:57:57,383 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:57,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:57:57,384 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T14:57:57,386 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740 2024-12-03T14:57:57,389 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740 2024-12-03T14:57:57,391 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T14:57:57,391 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T14:57:57,392 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T14:57:57,395 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T14:57:57,396 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72937813, jitterRate=0.0868581086397171}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T14:57:57,396 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T14:57:57,398 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733237877373Writing region info on filesystem at 1733237877373Initializing all the Stores at 1733237877374 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237877375 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237877375Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237877375Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237877375Cleaning up temporary data from old regions at 1733237877391 (+16 ms)Running coprocessor post-open hooks at 1733237877396 (+5 ms)Region opened successfully at 1733237877398 (+2 ms) 2024-12-03T14:57:57,404 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733237877311 2024-12-03T14:57:57,415 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T14:57:57,416 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T14:57:57,417 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:57:57,420 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,45227,1733237844286, state=OPEN 2024-12-03T14:57:57,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T14:57:57,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T14:57:57,423 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:57:57,423 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:57:57,423 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a5d22df9eca2,45227,1733237844286 2024-12-03T14:57:57,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T14:57:57,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,45227,1733237844286 in 283 msec 2024-12-03T14:57:57,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T14:57:57,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.7580 sec 2024-12-03T14:57:57,438 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T14:57:57,438 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T14:57:57,458 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:57:57,460 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,45227,1733237844286, seqNum=-1] 2024-12-03T14:57:57,484 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:57:57,487 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53775, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:57:57,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.7970 sec 2024-12-03T14:57:57,509 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733237877509, completionTime=-1 2024-12-03T14:57:57,511 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T14:57:57,511 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T14:57:57,533 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T14:57:57,533 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733237937533 2024-12-03T14:57:57,533 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733237997533 2024-12-03T14:57:57,533 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-12-03T14:57:57,535 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,44239,1733237843695-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:57,535 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,44239,1733237843695-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:57,535 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,44239,1733237843695-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:57,537 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a5d22df9eca2:44239, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:57,537 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:57,537 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:57,543 DEBUG [master/a5d22df9eca2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T14:57:57,560 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.157sec 2024-12-03T14:57:57,561 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T14:57:57,562 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T14:57:57,563 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T14:57:57,563 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T14:57:57,564 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T14:57:57,564 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,44239,1733237843695-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T14:57:57,565 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,44239,1733237843695-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T14:57:57,572 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T14:57:57,573 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T14:57:57,574 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,44239,1733237843695-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:57:57,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dded46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:57:57,611 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T14:57:57,611 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T14:57:57,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a5d22df9eca2,44239,-1 for getting cluster id 2024-12-03T14:57:57,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:57:57,623 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1d8f4de9-4ea3-498e-9402-5f1d161c99d2' 2024-12-03T14:57:57,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:57:57,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1d8f4de9-4ea3-498e-9402-5f1d161c99d2" 2024-12-03T14:57:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d3f08f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:57:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a5d22df9eca2,44239,-1] 2024-12-03T14:57:57,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:57:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:57:57,634 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:57:57,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76731d3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:57:57,637 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:57:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,45227,1733237844286, seqNum=-1] 2024-12-03T14:57:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:57:57,646 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:57:57,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a5d22df9eca2,44239,1733237843695 2024-12-03T14:57:57,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15adb061, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:57:57,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a5d22df9eca2,44239,-1 for getting cluster id 2024-12-03T14:57:57,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:57:57,667 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1d8f4de9-4ea3-498e-9402-5f1d161c99d2' 2024-12-03T14:57:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:57:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1d8f4de9-4ea3-498e-9402-5f1d161c99d2" 2024-12-03T14:57:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@200ed736, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:57:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a5d22df9eca2,44239,-1] 2024-12-03T14:57:57,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:57:57,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:57:57,671 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:57:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e5c623, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:57:57,682 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=311, ProcessCount=11, AvailableMemoryMB=5112 2024-12-03T14:57:57,688 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T14:57:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a5d22df9eca2,44239,1733237843695 2024-12-03T14:57:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@53a284bf 2024-12-03T14:57:57,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T14:57:57,699 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T14:57:57,705 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:57:57,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T14:57:57,715 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:57:57,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-03T14:57:57,717 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:57,736 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:57:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T14:57:57,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741835_1011 (size=428) 2024-12-03T14:57:58,160 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 37ac058b85763227b939db7883e3d341, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e 2024-12-03T14:57:58,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741836_1012 (size=79) 2024-12-03T14:57:58,574 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:57:58,575 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 37ac058b85763227b939db7883e3d341, disabling compactions & flushes 2024-12-03T14:57:58,575 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. 2024-12-03T14:57:58,575 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. 2024-12-03T14:57:58,575 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. after waiting 0 ms 2024-12-03T14:57:58,575 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. 2024-12-03T14:57:58,576 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. 2024-12-03T14:57:58,576 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 37ac058b85763227b939db7883e3d341: Waiting for close lock at 1733237878575Disabling compacts and flushes for region at 1733237878575Disabling writes for close at 1733237878575Writing region close event to WAL at 1733237878575Closed at 1733237878575 2024-12-03T14:57:58,581 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:57:58,587 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733237878581"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237878581"}]},"ts":"1733237878581"} 2024-12-03T14:57:58,592 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:57:58,594 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:57:58,596 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237878594"}]},"ts":"1733237878594"} 2024-12-03T14:57:58,603 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-03T14:57:58,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=37ac058b85763227b939db7883e3d341, ASSIGN}] 2024-12-03T14:57:58,607 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=37ac058b85763227b939db7883e3d341, ASSIGN 2024-12-03T14:57:58,608 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=37ac058b85763227b939db7883e3d341, ASSIGN; state=OFFLINE, location=a5d22df9eca2,45227,1733237844286; forceNewPlan=false, retain=false 2024-12-03T14:57:58,760 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=37ac058b85763227b939db7883e3d341, regionState=OPENING, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:57:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T14:57:58,764 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=37ac058b85763227b939db7883e3d341, ASSIGN because future has completed 2024-12-03T14:57:58,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37ac058b85763227b939db7883e3d341, server=a5d22df9eca2,45227,1733237844286}] 2024-12-03T14:57:58,932 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. 2024-12-03T14:57:58,932 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 37ac058b85763227b939db7883e3d341, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:57:58,933 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 37ac058b85763227b939db7883e3d341 2024-12-03T14:57:58,933 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:57:58,933 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 37ac058b85763227b939db7883e3d341 2024-12-03T14:57:58,933 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 37ac058b85763227b939db7883e3d341 2024-12-03T14:57:58,936 INFO [StoreOpener-37ac058b85763227b939db7883e3d341-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 37ac058b85763227b939db7883e3d341 2024-12-03T14:57:58,938 INFO [StoreOpener-37ac058b85763227b939db7883e3d341-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37ac058b85763227b939db7883e3d341 columnFamilyName testFamily 2024-12-03T14:57:58,939 DEBUG [StoreOpener-37ac058b85763227b939db7883e3d341-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:57:58,940 INFO [StoreOpener-37ac058b85763227b939db7883e3d341-1 {}] regionserver.HStore(327): Store=37ac058b85763227b939db7883e3d341/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:57:58,940 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 37ac058b85763227b939db7883e3d341 2024-12-03T14:57:58,942 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/37ac058b85763227b939db7883e3d341 2024-12-03T14:57:58,942 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/37ac058b85763227b939db7883e3d341 2024-12-03T14:57:58,943 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 37ac058b85763227b939db7883e3d341 2024-12-03T14:57:58,943 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 37ac058b85763227b939db7883e3d341 2024-12-03T14:57:58,945 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 37ac058b85763227b939db7883e3d341 2024-12-03T14:57:58,949 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/37ac058b85763227b939db7883e3d341/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:57:58,949 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 37ac058b85763227b939db7883e3d341; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59565534, jitterRate=-0.11240437626838684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:57:58,949 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 37ac058b85763227b939db7883e3d341 2024-12-03T14:57:58,950 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 37ac058b85763227b939db7883e3d341: Running coprocessor pre-open hook at 1733237878933Writing region info on filesystem at 1733237878933Initializing all the Stores at 1733237878935 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237878935Cleaning up temporary data from old regions at 1733237878943 (+8 ms)Running coprocessor post-open hooks at 1733237878950 (+7 ms)Region opened successfully at 1733237878950 2024-12-03T14:57:58,952 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341., pid=6, masterSystemTime=1733237878921 2024-12-03T14:57:58,956 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. 2024-12-03T14:57:58,956 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. 2024-12-03T14:57:58,957 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=37ac058b85763227b939db7883e3d341, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:57:58,961 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37ac058b85763227b939db7883e3d341, server=a5d22df9eca2,45227,1733237844286 because future has completed 2024-12-03T14:57:58,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T14:57:58,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 37ac058b85763227b939db7883e3d341, server=a5d22df9eca2,45227,1733237844286 in 197 msec 2024-12-03T14:57:58,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T14:57:58,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=37ac058b85763227b939db7883e3d341, ASSIGN in 363 msec 2024-12-03T14:57:58,973 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:57:58,974 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237878973"}]},"ts":"1733237878973"} 2024-12-03T14:57:58,977 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-03T14:57:58,978 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:57:58,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2710 sec 2024-12-03T14:58:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T14:58:00,787 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-03T14:58:00,787 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-03T14:58:00,790 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:00,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-03T14:58:00,795 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:00,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-03T14:58:00,798 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:58:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,45227,1733237844286, seqNum=-1] 2024-12-03T14:58:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:58:00,803 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:58:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2] 2024-12-03T14:58:00,825 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T14:58:00,828 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-03T14:58:00,833 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-03T14:58:00,837 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-03T14:58:00,839 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-03T14:58:00,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6423078111759302648 2024-12-03T14:58:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53142 deadline: 1733237883839, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T14:58:00,865 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-03T14:58:00,866 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-03T14:58:00,867 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T14:58:00,869 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T14:58:00,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6423078111759302646 2024-12-03T14:58:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53142 deadline: 1733237883869, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T14:58:00,874 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T14:58:00,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-6423078111759302645 2024-12-03T14:58:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53142 deadline: 1733237883874, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T14:58:00,878 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T14:58:00,879 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-6423078111759302644 2024-12-03T14:58:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53142 deadline: 1733237883878, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T14:58:00,883 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T14:58:00,894 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1621907427_22 at /127.0.0.1:48726 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=311 (was 311), ProcessCount=11 (was 11), AvailableMemoryMB=5055 (was 5112) 2024-12-03T14:58:00,903 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=311, ProcessCount=11, AvailableMemoryMB=5053 2024-12-03T14:58:00,908 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:58:00,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44239 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-03T14:58:00,913 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:58:00,914 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:00,914 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-03T14:58:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T14:58:00,916 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:58:00,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741837_1013 (size=414) 2024-12-03T14:58:01,330 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e50ad728224bb2567237109c0c01214d, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e 2024-12-03T14:58:01,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741838_1014 (size=65) 2024-12-03T14:58:01,740 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:01,740 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing e50ad728224bb2567237109c0c01214d, disabling compactions & flushes 2024-12-03T14:58:01,740 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. 2024-12-03T14:58:01,740 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. 2024-12-03T14:58:01,740 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. after waiting 0 ms 2024-12-03T14:58:01,741 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. 2024-12-03T14:58:01,741 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. 2024-12-03T14:58:01,741 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for e50ad728224bb2567237109c0c01214d: Waiting for close lock at 1733237881740Disabling compacts and flushes for region at 1733237881740Disabling writes for close at 1733237881741 (+1 ms)Writing region close event to WAL at 1733237881741Closed at 1733237881741 2024-12-03T14:58:01,743 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:58:01,743 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733237881743"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237881743"}]},"ts":"1733237881743"} 2024-12-03T14:58:01,747 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:58:01,749 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:58:01,749 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237881749"}]},"ts":"1733237881749"} 2024-12-03T14:58:01,752 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-03T14:58:01,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e50ad728224bb2567237109c0c01214d, ASSIGN}] 2024-12-03T14:58:01,755 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e50ad728224bb2567237109c0c01214d, ASSIGN 2024-12-03T14:58:01,757 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e50ad728224bb2567237109c0c01214d, ASSIGN; state=OFFLINE, location=a5d22df9eca2,45227,1733237844286; forceNewPlan=false, retain=false 2024-12-03T14:58:01,908 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e50ad728224bb2567237109c0c01214d, regionState=OPENING, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:01,912 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e50ad728224bb2567237109c0c01214d, ASSIGN because future has completed 2024-12-03T14:58:01,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e50ad728224bb2567237109c0c01214d, server=a5d22df9eca2,45227,1733237844286}] 2024-12-03T14:58:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T14:58:02,074 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. 2024-12-03T14:58:02,074 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => e50ad728224bb2567237109c0c01214d, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:58:02,075 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner e50ad728224bb2567237109c0c01214d 2024-12-03T14:58:02,075 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:02,075 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for e50ad728224bb2567237109c0c01214d 2024-12-03T14:58:02,075 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for e50ad728224bb2567237109c0c01214d 2024-12-03T14:58:02,077 INFO [StoreOpener-e50ad728224bb2567237109c0c01214d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e50ad728224bb2567237109c0c01214d 2024-12-03T14:58:02,080 INFO [StoreOpener-e50ad728224bb2567237109c0c01214d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e50ad728224bb2567237109c0c01214d columnFamilyName testFamily 2024-12-03T14:58:02,080 DEBUG [StoreOpener-e50ad728224bb2567237109c0c01214d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:02,081 INFO [StoreOpener-e50ad728224bb2567237109c0c01214d-1 {}] regionserver.HStore(327): Store=e50ad728224bb2567237109c0c01214d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:58:02,081 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for e50ad728224bb2567237109c0c01214d 2024-12-03T14:58:02,083 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScanner/e50ad728224bb2567237109c0c01214d 2024-12-03T14:58:02,084 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScanner/e50ad728224bb2567237109c0c01214d 2024-12-03T14:58:02,084 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for e50ad728224bb2567237109c0c01214d 2024-12-03T14:58:02,084 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for e50ad728224bb2567237109c0c01214d 2024-12-03T14:58:02,087 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for e50ad728224bb2567237109c0c01214d 2024-12-03T14:58:02,091 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScanner/e50ad728224bb2567237109c0c01214d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:58:02,092 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened e50ad728224bb2567237109c0c01214d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64510216, jitterRate=-0.038722872734069824}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:58:02,092 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e50ad728224bb2567237109c0c01214d 2024-12-03T14:58:02,093 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for e50ad728224bb2567237109c0c01214d: Running coprocessor pre-open hook at 1733237882075Writing region info on filesystem at 1733237882075Initializing all the Stores at 1733237882077 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237882077Cleaning up temporary data from old regions at 1733237882084 (+7 ms)Running coprocessor post-open hooks at 1733237882092 (+8 ms)Region opened successfully at 1733237882093 (+1 ms) 2024-12-03T14:58:02,095 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d., pid=9, masterSystemTime=1733237882068 2024-12-03T14:58:02,102 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. 2024-12-03T14:58:02,102 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. 2024-12-03T14:58:02,103 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e50ad728224bb2567237109c0c01214d, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:02,108 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e50ad728224bb2567237109c0c01214d, server=a5d22df9eca2,45227,1733237844286 because future has completed 2024-12-03T14:58:02,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-03T14:58:02,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure e50ad728224bb2567237109c0c01214d, server=a5d22df9eca2,45227,1733237844286 in 197 msec 2024-12-03T14:58:02,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T14:58:02,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e50ad728224bb2567237109c0c01214d, ASSIGN in 361 msec 2024-12-03T14:58:02,120 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:58:02,120 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237882120"}]},"ts":"1733237882120"} 2024-12-03T14:58:02,123 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-03T14:58:02,125 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:58:02,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2170 sec 2024-12-03T14:58:02,908 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:58:02,941 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-03T14:58:02,943 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T14:58:02,944 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-03T14:58:03,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T14:58:03,962 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-03T14:58:03,962 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-03T14:58:03,962 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:03,967 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-03T14:58:03,967 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:03,967 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-03T14:58:03,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2] 2024-12-03T14:58:03,986 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T14:58:03,986 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-03T14:58:03,991 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T14:58:03,993 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-03T14:58:04,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T14:58:04,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T14:58:04,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T14:58:04,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T14:58:04,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T14:58:04,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T14:58:04,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-03T14:58:04,043 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-03T14:58:10,005 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a5d22df9eca2:45227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-12-03T14:58:10,006 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-12-03T14:58:10,006 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 from cache 2024-12-03T14:58:10,016 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=347 (was 311) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5040 (was 5053) 2024-12-03T14:58:10,024 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=347, ProcessCount=11, AvailableMemoryMB=5039 2024-12-03T14:58:10,027 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:58:10,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44239 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-03T14:58:10,031 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:58:10,031 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:10,032 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-03T14:58:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T14:58:10,033 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:58:10,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741839_1015 (size=412) 2024-12-03T14:58:10,045 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ef7b07aadb8991db2c582b540fe7b844, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e 2024-12-03T14:58:10,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741840_1016 (size=63) 2024-12-03T14:58:10,457 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:10,457 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing ef7b07aadb8991db2c582b540fe7b844, disabling compactions & flushes 2024-12-03T14:58:10,457 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. 2024-12-03T14:58:10,457 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. 2024-12-03T14:58:10,457 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. after waiting 0 ms 2024-12-03T14:58:10,457 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. 2024-12-03T14:58:10,457 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. 2024-12-03T14:58:10,457 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for ef7b07aadb8991db2c582b540fe7b844: Waiting for close lock at 1733237890457Disabling compacts and flushes for region at 1733237890457Disabling writes for close at 1733237890457Writing region close event to WAL at 1733237890457Closed at 1733237890457 2024-12-03T14:58:10,459 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:58:10,460 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733237890459"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237890459"}]},"ts":"1733237890459"} 2024-12-03T14:58:10,463 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:58:10,464 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:58:10,465 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237890464"}]},"ts":"1733237890464"} 2024-12-03T14:58:10,467 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-03T14:58:10,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=ef7b07aadb8991db2c582b540fe7b844, ASSIGN}] 2024-12-03T14:58:10,470 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=ef7b07aadb8991db2c582b540fe7b844, ASSIGN 2024-12-03T14:58:10,471 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=ef7b07aadb8991db2c582b540fe7b844, ASSIGN; state=OFFLINE, location=a5d22df9eca2,45227,1733237844286; forceNewPlan=false, retain=false 2024-12-03T14:58:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-12-03T14:58:10,622 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=ef7b07aadb8991db2c582b540fe7b844, regionState=OPENING, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:10,626 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=ef7b07aadb8991db2c582b540fe7b844, ASSIGN because future has completed 2024-12-03T14:58:10,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7b07aadb8991db2c582b540fe7b844, server=a5d22df9eca2,45227,1733237844286}] 2024-12-03T14:58:10,787 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. 2024-12-03T14:58:10,787 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => ef7b07aadb8991db2c582b540fe7b844, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:58:10,788 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:10,788 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:10,788 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:10,788 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:10,791 INFO [StoreOpener-ef7b07aadb8991db2c582b540fe7b844-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:10,793 INFO [StoreOpener-ef7b07aadb8991db2c582b540fe7b844-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef7b07aadb8991db2c582b540fe7b844 columnFamilyName testFamily 2024-12-03T14:58:10,793 DEBUG [StoreOpener-ef7b07aadb8991db2c582b540fe7b844-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:10,794 INFO [StoreOpener-ef7b07aadb8991db2c582b540fe7b844-1 {}] regionserver.HStore(327): Store=ef7b07aadb8991db2c582b540fe7b844/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:58:10,794 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:10,796 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNextAsync/ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:10,797 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNextAsync/ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:10,797 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:10,797 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:10,800 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:10,804 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNextAsync/ef7b07aadb8991db2c582b540fe7b844/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:58:10,805 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened ef7b07aadb8991db2c582b540fe7b844; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70817624, jitterRate=0.055264830589294434}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:58:10,805 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:10,806 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for ef7b07aadb8991db2c582b540fe7b844: Running coprocessor pre-open hook at 1733237890788Writing region info on filesystem at 1733237890788Initializing all the Stores at 1733237890790 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237890790Cleaning up temporary data from old regions at 1733237890797 (+7 ms)Running coprocessor post-open hooks at 1733237890805 (+8 ms)Region opened successfully at 1733237890806 (+1 ms) 2024-12-03T14:58:10,808 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844., pid=12, masterSystemTime=1733237890780 2024-12-03T14:58:10,811 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. 2024-12-03T14:58:10,811 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. 2024-12-03T14:58:10,812 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=ef7b07aadb8991db2c582b540fe7b844, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:10,815 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7b07aadb8991db2c582b540fe7b844, server=a5d22df9eca2,45227,1733237844286 because future has completed 2024-12-03T14:58:10,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T14:58:10,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure ef7b07aadb8991db2c582b540fe7b844, server=a5d22df9eca2,45227,1733237844286 in 190 msec 2024-12-03T14:58:10,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-03T14:58:10,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=ef7b07aadb8991db2c582b540fe7b844, ASSIGN in 353 msec 2024-12-03T14:58:10,825 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:58:10,825 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237890825"}]},"ts":"1733237890825"} 2024-12-03T14:58:10,828 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-03T14:58:10,830 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:58:10,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 803 msec 2024-12-03T14:58:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T14:58:11,051 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-03T14:58:11,051 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-03T14:58:11,051 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:11,056 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-03T14:58:11,056 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:11,056 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-03T14:58:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2] 2024-12-03T14:58:11,074 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T14:58:11,074 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-03T14:58:11,079 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T14:58:11,082 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T14:58:11,083 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T14:58:11,084 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-03T14:58:14,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-03T14:58:14,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T14:58:14,946 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-03T14:58:16,314 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:58:17,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45227 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6423078111759302635' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T14:58:17,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45227 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:53142 deadline: 1733237903090, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6423078111759302635' does not exist 2024-12-03T14:58:17,096 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to a5d22df9eca2,45227,1733237844286 for closing scanner id = -6423078111759302635 for ef7b07aadb8991db2c582b540fe7b844 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6423078111759302635' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:58:17,100 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=371 (was 347) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5024 (was 5039) 2024-12-03T14:58:17,108 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=371, ProcessCount=11, AvailableMemoryMB=5024 2024-12-03T14:58:17,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:58:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-03T14:58:17,115 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:58:17,115 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:17,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-03T14:58:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T14:58:17,117 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:58:17,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741841_1017 (size=423) 2024-12-03T14:58:17,529 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 57c89a40077913e3c2dae3b7512725f9, NAME => 'testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e 2024-12-03T14:58:17,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741842_1018 (size=74) 2024-12-03T14:58:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-12-03T14:58:17,937 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:17,937 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 57c89a40077913e3c2dae3b7512725f9, disabling compactions & flushes 2024-12-03T14:58:17,937 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. 2024-12-03T14:58:17,937 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. 2024-12-03T14:58:17,938 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. after waiting 0 ms 2024-12-03T14:58:17,938 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. 2024-12-03T14:58:17,938 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. 2024-12-03T14:58:17,938 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 57c89a40077913e3c2dae3b7512725f9: Waiting for close lock at 1733237897937Disabling compacts and flushes for region at 1733237897937Disabling writes for close at 1733237897938 (+1 ms)Writing region close event to WAL at 1733237897938Closed at 1733237897938 2024-12-03T14:58:17,940 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:58:17,940 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733237897940"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237897940"}]},"ts":"1733237897940"} 2024-12-03T14:58:17,943 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:58:17,945 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:58:17,945 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237897945"}]},"ts":"1733237897945"} 2024-12-03T14:58:17,948 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-03T14:58:17,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=57c89a40077913e3c2dae3b7512725f9, ASSIGN}] 2024-12-03T14:58:17,950 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=57c89a40077913e3c2dae3b7512725f9, ASSIGN 2024-12-03T14:58:17,952 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=57c89a40077913e3c2dae3b7512725f9, ASSIGN; state=OFFLINE, location=a5d22df9eca2,45227,1733237844286; forceNewPlan=false, retain=false 2024-12-03T14:58:18,103 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=57c89a40077913e3c2dae3b7512725f9, regionState=OPENING, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:18,106 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=57c89a40077913e3c2dae3b7512725f9, ASSIGN because future has completed 2024-12-03T14:58:18,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57c89a40077913e3c2dae3b7512725f9, server=a5d22df9eca2,45227,1733237844286}] 2024-12-03T14:58:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T14:58:18,265 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. 2024-12-03T14:58:18,266 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 57c89a40077913e3c2dae3b7512725f9, NAME => 'testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:58:18,266 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:58:18,266 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:18,266 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:58:18,266 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:58:18,269 INFO [StoreOpener-57c89a40077913e3c2dae3b7512725f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:58:18,273 INFO [StoreOpener-57c89a40077913e3c2dae3b7512725f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57c89a40077913e3c2dae3b7512725f9 columnFamilyName testFamily 2024-12-03T14:58:18,273 DEBUG [StoreOpener-57c89a40077913e3c2dae3b7512725f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:18,274 INFO [StoreOpener-57c89a40077913e3c2dae3b7512725f9-1 {}] regionserver.HStore(327): Store=57c89a40077913e3c2dae3b7512725f9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:58:18,274 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:58:18,275 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextException/57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:58:18,275 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextException/57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:58:18,276 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:58:18,276 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:58:18,279 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:58:18,282 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextException/57c89a40077913e3c2dae3b7512725f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:58:18,283 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 57c89a40077913e3c2dae3b7512725f9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65110452, jitterRate=-0.029778659343719482}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:58:18,284 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:58:18,284 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 57c89a40077913e3c2dae3b7512725f9: Running coprocessor pre-open hook at 1733237898266Writing region info on filesystem at 1733237898266Initializing all the Stores at 1733237898268 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237898268Cleaning up temporary data from old regions at 1733237898276 (+8 ms)Running coprocessor post-open hooks at 1733237898284 (+8 ms)Region opened successfully at 1733237898284 2024-12-03T14:58:18,286 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9., pid=15, masterSystemTime=1733237898260 2024-12-03T14:58:18,289 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. 2024-12-03T14:58:18,289 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. 2024-12-03T14:58:18,290 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=57c89a40077913e3c2dae3b7512725f9, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:18,293 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57c89a40077913e3c2dae3b7512725f9, server=a5d22df9eca2,45227,1733237844286 because future has completed 2024-12-03T14:58:18,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-03T14:58:18,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 57c89a40077913e3c2dae3b7512725f9, server=a5d22df9eca2,45227,1733237844286 in 188 msec 2024-12-03T14:58:18,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T14:58:18,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=57c89a40077913e3c2dae3b7512725f9, ASSIGN in 351 msec 2024-12-03T14:58:18,303 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:58:18,303 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237898303"}]},"ts":"1733237898303"} 2024-12-03T14:58:18,309 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-03T14:58:18,310 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:58:18,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.1980 sec 2024-12-03T14:58:20,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T14:58:20,151 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-03T14:58:20,151 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-03T14:58:20,151 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:20,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-03T14:58:20,156 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:20,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-03T14:58:20,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2] 2024-12-03T14:58:20,172 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T14:58:20,173 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-03T14:58:20,177 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-03T14:58:20,179 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-03T14:58:20,180 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-03T14:58:20,181 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6423078111759302631 2024-12-03T14:58:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53142 deadline: 1733237903180, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T14:58:20,185 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-03T14:58:20,187 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-03T14:58:20,188 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T14:58:20,189 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T14:58:20,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6423078111759302629 2024-12-03T14:58:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53142 deadline: 1733237903189, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T14:58:20,193 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T14:58:20,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-6423078111759302628 2024-12-03T14:58:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53142 deadline: 1733237903193, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T14:58:20,197 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T14:58:20,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-6423078111759302627 2024-12-03T14:58:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53142 deadline: 1733237903197, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T14:58:20,202 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T14:58:20,212 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=228 (was 225) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1621907427_22 at /127.0.0.1:40608 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=371 (was 371), ProcessCount=11 (was 11), AvailableMemoryMB=5019 (was 5024) 2024-12-03T14:58:20,221 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=228, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=371, ProcessCount=11, AvailableMemoryMB=5019 2024-12-03T14:58:20,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:58:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-03T14:58:20,226 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:58:20,227 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:20,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-03T14:58:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T14:58:20,228 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:58:20,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741843_1019 (size=418) 2024-12-03T14:58:20,640 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b41aac3f4066a49ef9332272456c1a99, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e 2024-12-03T14:58:20,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741844_1020 (size=69) 2024-12-03T14:58:20,652 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:20,652 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing b41aac3f4066a49ef9332272456c1a99, disabling compactions & flushes 2024-12-03T14:58:20,652 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. 2024-12-03T14:58:20,652 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. 2024-12-03T14:58:20,652 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. after waiting 0 ms 2024-12-03T14:58:20,652 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. 2024-12-03T14:58:20,652 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. 2024-12-03T14:58:20,652 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for b41aac3f4066a49ef9332272456c1a99: Waiting for close lock at 1733237900652Disabling compacts and flushes for region at 1733237900652Disabling writes for close at 1733237900652Writing region close event to WAL at 1733237900652Closed at 1733237900652 2024-12-03T14:58:20,654 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:58:20,654 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733237900654"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237900654"}]},"ts":"1733237900654"} 2024-12-03T14:58:20,657 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:58:20,659 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:58:20,659 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237900659"}]},"ts":"1733237900659"} 2024-12-03T14:58:20,662 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-03T14:58:20,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=b41aac3f4066a49ef9332272456c1a99, ASSIGN}] 2024-12-03T14:58:20,664 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=b41aac3f4066a49ef9332272456c1a99, ASSIGN 2024-12-03T14:58:20,666 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=b41aac3f4066a49ef9332272456c1a99, ASSIGN; state=OFFLINE, location=a5d22df9eca2,45227,1733237844286; forceNewPlan=false, retain=false 2024-12-03T14:58:20,817 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=b41aac3f4066a49ef9332272456c1a99, regionState=OPENING, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:20,820 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=b41aac3f4066a49ef9332272456c1a99, ASSIGN because future has completed 2024-12-03T14:58:20,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure b41aac3f4066a49ef9332272456c1a99, server=a5d22df9eca2,45227,1733237844286}] 2024-12-03T14:58:20,947 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-03T14:58:20,978 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. 2024-12-03T14:58:20,979 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => b41aac3f4066a49ef9332272456c1a99, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:58:20,979 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner b41aac3f4066a49ef9332272456c1a99 2024-12-03T14:58:20,979 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:20,979 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for b41aac3f4066a49ef9332272456c1a99 2024-12-03T14:58:20,979 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for b41aac3f4066a49ef9332272456c1a99 2024-12-03T14:58:20,981 INFO [StoreOpener-b41aac3f4066a49ef9332272456c1a99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b41aac3f4066a49ef9332272456c1a99 2024-12-03T14:58:20,984 INFO [StoreOpener-b41aac3f4066a49ef9332272456c1a99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b41aac3f4066a49ef9332272456c1a99 columnFamilyName testFamily 2024-12-03T14:58:20,984 DEBUG [StoreOpener-b41aac3f4066a49ef9332272456c1a99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:20,985 INFO [StoreOpener-b41aac3f4066a49ef9332272456c1a99-1 {}] regionserver.HStore(327): Store=b41aac3f4066a49ef9332272456c1a99/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:58:20,985 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for b41aac3f4066a49ef9332272456c1a99 2024-12-03T14:58:20,987 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScanner/b41aac3f4066a49ef9332272456c1a99 2024-12-03T14:58:20,988 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScanner/b41aac3f4066a49ef9332272456c1a99 2024-12-03T14:58:20,988 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for b41aac3f4066a49ef9332272456c1a99 2024-12-03T14:58:20,988 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for b41aac3f4066a49ef9332272456c1a99 2024-12-03T14:58:20,991 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for b41aac3f4066a49ef9332272456c1a99 2024-12-03T14:58:20,995 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScanner/b41aac3f4066a49ef9332272456c1a99/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:58:20,995 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened b41aac3f4066a49ef9332272456c1a99; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66715409, jitterRate=-0.00586293637752533}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:58:20,995 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b41aac3f4066a49ef9332272456c1a99 2024-12-03T14:58:20,996 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for b41aac3f4066a49ef9332272456c1a99: Running coprocessor pre-open hook at 1733237900979Writing region info on filesystem at 1733237900979Initializing all the Stores at 1733237900981 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237900981Cleaning up temporary data from old regions at 1733237900988 (+7 ms)Running coprocessor post-open hooks at 1733237900995 (+7 ms)Region opened successfully at 1733237900996 (+1 ms) 2024-12-03T14:58:20,997 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99., pid=18, masterSystemTime=1733237900973 2024-12-03T14:58:21,000 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. 2024-12-03T14:58:21,000 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. 2024-12-03T14:58:21,001 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=b41aac3f4066a49ef9332272456c1a99, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:21,003 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure b41aac3f4066a49ef9332272456c1a99, server=a5d22df9eca2,45227,1733237844286 because future has completed 2024-12-03T14:58:21,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T14:58:21,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure b41aac3f4066a49ef9332272456c1a99, server=a5d22df9eca2,45227,1733237844286 in 185 msec 2024-12-03T14:58:21,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-03T14:58:21,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=b41aac3f4066a49ef9332272456c1a99, ASSIGN in 345 msec 2024-12-03T14:58:21,012 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:58:21,013 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237901012"}]},"ts":"1733237901012"} 2024-12-03T14:58:21,015 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-03T14:58:21,016 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:58:21,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 793 msec 2024-12-03T14:58:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T14:58:21,242 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-03T14:58:21,242 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-03T14:58:21,243 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:21,253 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-03T14:58:21,254 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:21,254 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-03T14:58:21,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2] 2024-12-03T14:58:21,269 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T14:58:21,269 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-03T14:58:21,272 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T14:58:21,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T14:58:22,279 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a5d22df9eca2:45227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-12-03T14:58:22,279 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-12-03T14:58:22,279 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 from cache 2024-12-03T14:58:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-12-03T14:58:22,799 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T14:58:23,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2] 2024-12-03T14:58:23,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T14:58:23,772 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:58:24,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-03T14:58:24,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-03T14:58:24,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-03T14:58:24,044 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-03T14:58:24,279 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a5d22df9eca2:45227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=983ms, rpcTimeout=976ms 2024-12-03T14:58:24,279 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=983ms, rpcTimeout=976ms 2024-12-03T14:58:24,279 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 from cache 2024-12-03T14:58:24,294 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223 (was 228), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=341 (was 371), ProcessCount=11 (was 11), AvailableMemoryMB=5015 (was 5019) 2024-12-03T14:58:24,306 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=341, ProcessCount=11, AvailableMemoryMB=5014 2024-12-03T14:58:24,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:58:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-03T14:58:24,312 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:58:24,313 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:24,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-03T14:58:24,314 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:58:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T14:58:24,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741845_1021 (size=416) 2024-12-03T14:58:24,732 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 03bbcd680c9a9257b7bf3639be6cfe7a, NAME => 'testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e 2024-12-03T14:58:24,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741846_1022 (size=67) 2024-12-03T14:58:24,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-12-03T14:58:25,146 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:25,146 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 03bbcd680c9a9257b7bf3639be6cfe7a, disabling compactions & flushes 2024-12-03T14:58:25,147 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. 2024-12-03T14:58:25,147 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. 2024-12-03T14:58:25,147 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. after waiting 0 ms 2024-12-03T14:58:25,147 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. 2024-12-03T14:58:25,147 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. 2024-12-03T14:58:25,148 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 03bbcd680c9a9257b7bf3639be6cfe7a: Waiting for close lock at 1733237905146Disabling compacts and flushes for region at 1733237905146Disabling writes for close at 1733237905147 (+1 ms)Writing region close event to WAL at 1733237905147Closed at 1733237905147 2024-12-03T14:58:25,151 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:58:25,152 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733237905151"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237905151"}]},"ts":"1733237905151"} 2024-12-03T14:58:25,154 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:58:25,156 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:58:25,156 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237905156"}]},"ts":"1733237905156"} 2024-12-03T14:58:25,158 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-03T14:58:25,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=03bbcd680c9a9257b7bf3639be6cfe7a, ASSIGN}] 2024-12-03T14:58:25,160 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=03bbcd680c9a9257b7bf3639be6cfe7a, ASSIGN 2024-12-03T14:58:25,162 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=03bbcd680c9a9257b7bf3639be6cfe7a, ASSIGN; state=OFFLINE, location=a5d22df9eca2,45227,1733237844286; forceNewPlan=false, retain=false 2024-12-03T14:58:25,314 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=03bbcd680c9a9257b7bf3639be6cfe7a, regionState=OPENING, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:25,322 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=03bbcd680c9a9257b7bf3639be6cfe7a, ASSIGN because future has completed 2024-12-03T14:58:25,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03bbcd680c9a9257b7bf3639be6cfe7a, server=a5d22df9eca2,45227,1733237844286}] 2024-12-03T14:58:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T14:58:25,486 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. 2024-12-03T14:58:25,486 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 03bbcd680c9a9257b7bf3639be6cfe7a, NAME => 'testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:58:25,487 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 03bbcd680c9a9257b7bf3639be6cfe7a 2024-12-03T14:58:25,487 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:25,487 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 03bbcd680c9a9257b7bf3639be6cfe7a 2024-12-03T14:58:25,487 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 03bbcd680c9a9257b7bf3639be6cfe7a 2024-12-03T14:58:25,490 INFO [StoreOpener-03bbcd680c9a9257b7bf3639be6cfe7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 03bbcd680c9a9257b7bf3639be6cfe7a 2024-12-03T14:58:25,492 INFO [StoreOpener-03bbcd680c9a9257b7bf3639be6cfe7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03bbcd680c9a9257b7bf3639be6cfe7a columnFamilyName testFamily 2024-12-03T14:58:25,492 DEBUG [StoreOpener-03bbcd680c9a9257b7bf3639be6cfe7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:25,493 INFO [StoreOpener-03bbcd680c9a9257b7bf3639be6cfe7a-1 {}] regionserver.HStore(327): Store=03bbcd680c9a9257b7bf3639be6cfe7a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:58:25,493 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 03bbcd680c9a9257b7bf3639be6cfe7a 2024-12-03T14:58:25,494 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNextAsync/03bbcd680c9a9257b7bf3639be6cfe7a 2024-12-03T14:58:25,495 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNextAsync/03bbcd680c9a9257b7bf3639be6cfe7a 2024-12-03T14:58:25,496 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 03bbcd680c9a9257b7bf3639be6cfe7a 2024-12-03T14:58:25,496 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 03bbcd680c9a9257b7bf3639be6cfe7a 2024-12-03T14:58:25,499 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 03bbcd680c9a9257b7bf3639be6cfe7a 2024-12-03T14:58:25,503 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNextAsync/03bbcd680c9a9257b7bf3639be6cfe7a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:58:25,504 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 03bbcd680c9a9257b7bf3639be6cfe7a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65084531, jitterRate=-0.030164912343025208}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:58:25,505 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 03bbcd680c9a9257b7bf3639be6cfe7a 2024-12-03T14:58:25,505 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 03bbcd680c9a9257b7bf3639be6cfe7a: Running coprocessor pre-open hook at 1733237905488Writing region info on filesystem at 1733237905488Initializing all the Stores at 1733237905489 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237905489Cleaning up temporary data from old regions at 1733237905496 (+7 ms)Running coprocessor post-open hooks at 1733237905505 (+9 ms)Region opened successfully at 1733237905505 2024-12-03T14:58:25,506 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a., pid=21, masterSystemTime=1733237905475 2024-12-03T14:58:25,509 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. 2024-12-03T14:58:25,509 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. 2024-12-03T14:58:25,510 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=03bbcd680c9a9257b7bf3639be6cfe7a, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:25,512 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03bbcd680c9a9257b7bf3639be6cfe7a, server=a5d22df9eca2,45227,1733237844286 because future has completed 2024-12-03T14:58:25,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-03T14:58:25,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 03bbcd680c9a9257b7bf3639be6cfe7a, server=a5d22df9eca2,45227,1733237844286 in 191 msec 2024-12-03T14:58:25,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T14:58:25,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=03bbcd680c9a9257b7bf3639be6cfe7a, ASSIGN in 357 msec 2024-12-03T14:58:25,520 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:58:25,520 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237905520"}]},"ts":"1733237905520"} 2024-12-03T14:58:25,522 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-03T14:58:25,523 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:58:25,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2140 sec 2024-12-03T14:58:26,948 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-03T14:58:26,950 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-03T14:58:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T14:58:27,362 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-03T14:58:27,362 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-03T14:58:27,362 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:27,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-03T14:58:27,367 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:27,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-03T14:58:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2] 2024-12-03T14:58:27,383 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T14:58:27,383 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-03T14:58:27,387 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T14:58:27,390 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T14:58:27,391 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T14:58:27,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-03T14:58:30,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45227 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6423078111759302616' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T14:58:30,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45227 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:53142 deadline: 1733237911401, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6423078111759302616' does not exist 2024-12-03T14:58:30,402 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to a5d22df9eca2,45227,1733237844286 for closing scanner id = -6423078111759302616 for 03bbcd680c9a9257b7bf3639be6cfe7a of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6423078111759302616' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:58:30,407 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 223), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=314 (was 341), ProcessCount=11 (was 11), AvailableMemoryMB=5009 (was 5014) 2024-12-03T14:58:30,416 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=314, ProcessCount=11, AvailableMemoryMB=5009 2024-12-03T14:58:30,418 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:58:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-03T14:58:30,421 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:58:30,422 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:30,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-03T14:58:30,423 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:58:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T14:58:30,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741847_1023 (size=411) 2024-12-03T14:58:30,834 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4776bec4819b23bad8df61b64e7c38b7, NAME => 'testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e 2024-12-03T14:58:30,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741848_1024 (size=62) 2024-12-03T14:58:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-12-03T14:58:30,990 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:58:31,245 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:31,246 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 4776bec4819b23bad8df61b64e7c38b7, disabling compactions & flushes 2024-12-03T14:58:31,246 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. 2024-12-03T14:58:31,246 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. 2024-12-03T14:58:31,246 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. after waiting 0 ms 2024-12-03T14:58:31,246 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. 2024-12-03T14:58:31,247 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. 2024-12-03T14:58:31,247 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4776bec4819b23bad8df61b64e7c38b7: Waiting for close lock at 1733237911246Disabling compacts and flushes for region at 1733237911246Disabling writes for close at 1733237911246Writing region close event to WAL at 1733237911247 (+1 ms)Closed at 1733237911247 2024-12-03T14:58:31,251 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:58:31,252 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733237911251"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237911251"}]},"ts":"1733237911251"} 2024-12-03T14:58:31,256 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:58:31,257 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:58:31,258 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237911257"}]},"ts":"1733237911257"} 2024-12-03T14:58:31,261 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-03T14:58:31,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=4776bec4819b23bad8df61b64e7c38b7, ASSIGN}] 2024-12-03T14:58:31,263 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=4776bec4819b23bad8df61b64e7c38b7, ASSIGN 2024-12-03T14:58:31,264 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=4776bec4819b23bad8df61b64e7c38b7, ASSIGN; state=OFFLINE, location=a5d22df9eca2,45227,1733237844286; forceNewPlan=false, retain=false 2024-12-03T14:58:31,415 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4776bec4819b23bad8df61b64e7c38b7, regionState=OPENING, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:31,421 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=4776bec4819b23bad8df61b64e7c38b7, ASSIGN because future has completed 2024-12-03T14:58:31,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4776bec4819b23bad8df61b64e7c38b7, server=a5d22df9eca2,45227,1733237844286}] 2024-12-03T14:58:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T14:58:31,588 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. 2024-12-03T14:58:31,589 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 4776bec4819b23bad8df61b64e7c38b7, NAME => 'testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:58:31,589 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 4776bec4819b23bad8df61b64e7c38b7 2024-12-03T14:58:31,589 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:31,590 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 4776bec4819b23bad8df61b64e7c38b7 2024-12-03T14:58:31,590 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 4776bec4819b23bad8df61b64e7c38b7 2024-12-03T14:58:31,592 INFO [StoreOpener-4776bec4819b23bad8df61b64e7c38b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4776bec4819b23bad8df61b64e7c38b7 2024-12-03T14:58:31,595 INFO [StoreOpener-4776bec4819b23bad8df61b64e7c38b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4776bec4819b23bad8df61b64e7c38b7 columnFamilyName testFamily 2024-12-03T14:58:31,595 DEBUG [StoreOpener-4776bec4819b23bad8df61b64e7c38b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:31,596 INFO [StoreOpener-4776bec4819b23bad8df61b64e7c38b7-1 {}] regionserver.HStore(327): Store=4776bec4819b23bad8df61b64e7c38b7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:58:31,596 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 4776bec4819b23bad8df61b64e7c38b7 2024-12-03T14:58:31,598 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNext/4776bec4819b23bad8df61b64e7c38b7 2024-12-03T14:58:31,599 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNext/4776bec4819b23bad8df61b64e7c38b7 2024-12-03T14:58:31,600 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 4776bec4819b23bad8df61b64e7c38b7 2024-12-03T14:58:31,600 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 4776bec4819b23bad8df61b64e7c38b7 2024-12-03T14:58:31,603 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 4776bec4819b23bad8df61b64e7c38b7 2024-12-03T14:58:31,607 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNext/4776bec4819b23bad8df61b64e7c38b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:58:31,608 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 4776bec4819b23bad8df61b64e7c38b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65079324, jitterRate=-0.030242502689361572}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:58:31,608 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4776bec4819b23bad8df61b64e7c38b7 2024-12-03T14:58:31,608 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 4776bec4819b23bad8df61b64e7c38b7: Running coprocessor pre-open hook at 1733237911590Writing region info on filesystem at 1733237911590Initializing all the Stores at 1733237911592 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237911592Cleaning up temporary data from old regions at 1733237911600 (+8 ms)Running coprocessor post-open hooks at 1733237911608 (+8 ms)Region opened successfully at 1733237911608 2024-12-03T14:58:31,610 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7., pid=24, masterSystemTime=1733237911577 2024-12-03T14:58:31,613 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. 2024-12-03T14:58:31,613 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. 2024-12-03T14:58:31,614 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=4776bec4819b23bad8df61b64e7c38b7, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:31,616 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4776bec4819b23bad8df61b64e7c38b7, server=a5d22df9eca2,45227,1733237844286 because future has completed 2024-12-03T14:58:31,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-03T14:58:31,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 4776bec4819b23bad8df61b64e7c38b7, server=a5d22df9eca2,45227,1733237844286 in 195 msec 2024-12-03T14:58:31,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-03T14:58:31,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=4776bec4819b23bad8df61b64e7c38b7, ASSIGN in 359 msec 2024-12-03T14:58:31,623 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:58:31,624 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237911623"}]},"ts":"1733237911623"} 2024-12-03T14:58:31,626 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-03T14:58:31,627 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:58:31,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2090 sec 2024-12-03T14:58:32,953 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-03T14:58:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T14:58:33,464 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-03T14:58:33,464 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-03T14:58:33,465 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:33,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-03T14:58:33,474 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:33,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-03T14:58:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2] 2024-12-03T14:58:33,489 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T14:58:33,489 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-03T14:58:33,492 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T14:58:33,495 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T14:58:33,497 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T14:58:33,497 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-03T14:58:34,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-03T14:58:34,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-03T14:58:34,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-03T14:58:34,043 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T14:58:36,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45227 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6423078111759302612' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T14:58:36,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45227 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:53142 deadline: 1733237917506, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6423078111759302612' does not exist 2024-12-03T14:58:36,509 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to a5d22df9eca2,45227,1733237844286 for closing scanner id = -6423078111759302612 for 4776bec4819b23bad8df61b64e7c38b7 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6423078111759302612' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:58:36,516 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=289 (was 314), ProcessCount=11 (was 11), AvailableMemoryMB=5005 (was 5009) 2024-12-03T14:58:36,524 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=289, ProcessCount=11, AvailableMemoryMB=5005 2024-12-03T14:58:36,526 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:58:36,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44239 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T14:58:36,529 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:58:36,529 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:36,529 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-03T14:58:36,530 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:58:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T14:58:36,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741849_1025 (size=419) 2024-12-03T14:58:36,946 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ccf86b0ce002ced573170bfc41cbf8bf, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e 2024-12-03T14:58:36,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741850_1026 (size=70) 2024-12-03T14:58:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-12-03T14:58:37,092 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:58:37,359 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:37,359 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing ccf86b0ce002ced573170bfc41cbf8bf, disabling compactions & flushes 2024-12-03T14:58:37,360 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. 2024-12-03T14:58:37,360 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. 2024-12-03T14:58:37,360 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. after waiting 0 ms 2024-12-03T14:58:37,360 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. 2024-12-03T14:58:37,360 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. 2024-12-03T14:58:37,360 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for ccf86b0ce002ced573170bfc41cbf8bf: Waiting for close lock at 1733237917359Disabling compacts and flushes for region at 1733237917359Disabling writes for close at 1733237917360 (+1 ms)Writing region close event to WAL at 1733237917360Closed at 1733237917360 2024-12-03T14:58:37,365 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:58:37,365 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733237917365"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237917365"}]},"ts":"1733237917365"} 2024-12-03T14:58:37,369 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:58:37,371 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:58:37,371 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237917371"}]},"ts":"1733237917371"} 2024-12-03T14:58:37,374 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-03T14:58:37,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=ccf86b0ce002ced573170bfc41cbf8bf, ASSIGN}] 2024-12-03T14:58:37,376 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=ccf86b0ce002ced573170bfc41cbf8bf, ASSIGN 2024-12-03T14:58:37,377 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=ccf86b0ce002ced573170bfc41cbf8bf, ASSIGN; state=OFFLINE, location=a5d22df9eca2,45227,1733237844286; forceNewPlan=false, retain=false 2024-12-03T14:58:37,528 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=ccf86b0ce002ced573170bfc41cbf8bf, regionState=OPENING, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:37,531 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=ccf86b0ce002ced573170bfc41cbf8bf, ASSIGN because future has completed 2024-12-03T14:58:37,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccf86b0ce002ced573170bfc41cbf8bf, server=a5d22df9eca2,45227,1733237844286}] 2024-12-03T14:58:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T14:58:37,698 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. 2024-12-03T14:58:37,699 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => ccf86b0ce002ced573170bfc41cbf8bf, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:58:37,699 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync ccf86b0ce002ced573170bfc41cbf8bf 2024-12-03T14:58:37,699 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:37,699 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for ccf86b0ce002ced573170bfc41cbf8bf 2024-12-03T14:58:37,699 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for ccf86b0ce002ced573170bfc41cbf8bf 2024-12-03T14:58:37,702 INFO [StoreOpener-ccf86b0ce002ced573170bfc41cbf8bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ccf86b0ce002ced573170bfc41cbf8bf 2024-12-03T14:58:37,706 INFO [StoreOpener-ccf86b0ce002ced573170bfc41cbf8bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccf86b0ce002ced573170bfc41cbf8bf columnFamilyName testFamily 2024-12-03T14:58:37,707 DEBUG [StoreOpener-ccf86b0ce002ced573170bfc41cbf8bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:37,707 INFO [StoreOpener-ccf86b0ce002ced573170bfc41cbf8bf-1 {}] regionserver.HStore(327): Store=ccf86b0ce002ced573170bfc41cbf8bf/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:58:37,708 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for ccf86b0ce002ced573170bfc41cbf8bf 2024-12-03T14:58:37,709 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/ccf86b0ce002ced573170bfc41cbf8bf 2024-12-03T14:58:37,710 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/ccf86b0ce002ced573170bfc41cbf8bf 2024-12-03T14:58:37,710 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for ccf86b0ce002ced573170bfc41cbf8bf 2024-12-03T14:58:37,711 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for ccf86b0ce002ced573170bfc41cbf8bf 2024-12-03T14:58:37,713 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for ccf86b0ce002ced573170bfc41cbf8bf 2024-12-03T14:58:37,727 INFO [master/a5d22df9eca2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T14:58:37,727 INFO [master/a5d22df9eca2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T14:58:37,730 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/ccf86b0ce002ced573170bfc41cbf8bf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:58:37,731 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened ccf86b0ce002ced573170bfc41cbf8bf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59974662, jitterRate=-0.10630789399147034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:58:37,731 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ccf86b0ce002ced573170bfc41cbf8bf 2024-12-03T14:58:37,732 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for ccf86b0ce002ced573170bfc41cbf8bf: Running coprocessor pre-open hook at 1733237917700Writing region info on filesystem at 1733237917700Initializing all the Stores at 1733237917701 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237917701Cleaning up temporary data from old regions at 1733237917711 (+10 ms)Running coprocessor post-open hooks at 1733237917731 (+20 ms)Region opened successfully at 1733237917732 (+1 ms) 2024-12-03T14:58:37,733 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf., pid=27, masterSystemTime=1733237917693 2024-12-03T14:58:37,736 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. 2024-12-03T14:58:37,736 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. 2024-12-03T14:58:37,737 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=ccf86b0ce002ced573170bfc41cbf8bf, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:37,740 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccf86b0ce002ced573170bfc41cbf8bf, server=a5d22df9eca2,45227,1733237844286 because future has completed 2024-12-03T14:58:37,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-03T14:58:37,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure ccf86b0ce002ced573170bfc41cbf8bf, server=a5d22df9eca2,45227,1733237844286 in 210 msec 2024-12-03T14:58:37,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-03T14:58:37,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=ccf86b0ce002ced573170bfc41cbf8bf, ASSIGN in 372 msec 2024-12-03T14:58:37,751 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:58:37,751 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237917751"}]},"ts":"1733237917751"} 2024-12-03T14:58:37,754 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-03T14:58:37,756 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:58:37,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2300 sec 2024-12-03T14:58:38,955 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-03T14:58:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T14:58:39,561 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-03T14:58:39,561 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-03T14:58:39,561 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:39,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-03T14:58:39,566 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:39,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-03T14:58:39,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2] 2024-12-03T14:58:39,582 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T14:58:39,582 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-03T14:58:39,586 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T14:58:39,588 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-03T14:58:43,201 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:58:44,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T14:58:44,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T14:58:45,588 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a5d22df9eca2:45227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6001ms, rpcTimeout=5999ms 2024-12-03T14:58:45,589 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6001ms, rpcTimeout=5999ms 2024-12-03T14:58:45,589 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 from cache 2024-12-03T14:58:45,598 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=284 (was 289), ProcessCount=11 (was 11), AvailableMemoryMB=4992 (was 5005) 2024-12-03T14:58:45,606 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=284, ProcessCount=11, AvailableMemoryMB=4992 2024-12-03T14:58:45,608 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:58:45,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44239 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-03T14:58:45,611 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:58:45,611 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:45,611 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-03T14:58:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T14:58:45,612 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:58:45,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741851_1027 (size=407) 2024-12-03T14:58:46,024 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ae65c8dcc6508897b56b3fa8382fa1e5, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e 2024-12-03T14:58:46,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741852_1028 (size=58) 2024-12-03T14:58:46,036 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:46,036 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing ae65c8dcc6508897b56b3fa8382fa1e5, disabling compactions & flushes 2024-12-03T14:58:46,036 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. 2024-12-03T14:58:46,036 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. 2024-12-03T14:58:46,036 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. after waiting 0 ms 2024-12-03T14:58:46,036 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. 2024-12-03T14:58:46,036 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. 2024-12-03T14:58:46,036 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for ae65c8dcc6508897b56b3fa8382fa1e5: Waiting for close lock at 1733237926036Disabling compacts and flushes for region at 1733237926036Disabling writes for close at 1733237926036Writing region close event to WAL at 1733237926036Closed at 1733237926036 2024-12-03T14:58:46,038 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:58:46,038 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733237926038"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237926038"}]},"ts":"1733237926038"} 2024-12-03T14:58:46,041 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:58:46,042 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:58:46,043 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237926042"}]},"ts":"1733237926042"} 2024-12-03T14:58:46,045 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-03T14:58:46,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=ae65c8dcc6508897b56b3fa8382fa1e5, ASSIGN}] 2024-12-03T14:58:46,047 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=ae65c8dcc6508897b56b3fa8382fa1e5, ASSIGN 2024-12-03T14:58:46,048 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=ae65c8dcc6508897b56b3fa8382fa1e5, ASSIGN; state=OFFLINE, location=a5d22df9eca2,45227,1733237844286; forceNewPlan=false, retain=false 2024-12-03T14:58:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-12-03T14:58:46,199 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=ae65c8dcc6508897b56b3fa8382fa1e5, regionState=OPENING, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:46,205 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=ae65c8dcc6508897b56b3fa8382fa1e5, ASSIGN because future has completed 2024-12-03T14:58:46,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae65c8dcc6508897b56b3fa8382fa1e5, server=a5d22df9eca2,45227,1733237844286}] 2024-12-03T14:58:46,364 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. 2024-12-03T14:58:46,364 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => ae65c8dcc6508897b56b3fa8382fa1e5, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:58:46,365 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext ae65c8dcc6508897b56b3fa8382fa1e5 2024-12-03T14:58:46,365 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:46,365 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for ae65c8dcc6508897b56b3fa8382fa1e5 2024-12-03T14:58:46,365 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for ae65c8dcc6508897b56b3fa8382fa1e5 2024-12-03T14:58:46,367 INFO [StoreOpener-ae65c8dcc6508897b56b3fa8382fa1e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ae65c8dcc6508897b56b3fa8382fa1e5 2024-12-03T14:58:46,369 INFO [StoreOpener-ae65c8dcc6508897b56b3fa8382fa1e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae65c8dcc6508897b56b3fa8382fa1e5 columnFamilyName testFamily 2024-12-03T14:58:46,369 DEBUG [StoreOpener-ae65c8dcc6508897b56b3fa8382fa1e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:46,370 INFO [StoreOpener-ae65c8dcc6508897b56b3fa8382fa1e5-1 {}] regionserver.HStore(327): Store=ae65c8dcc6508897b56b3fa8382fa1e5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:58:46,370 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for ae65c8dcc6508897b56b3fa8382fa1e5 2024-12-03T14:58:46,371 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNext/ae65c8dcc6508897b56b3fa8382fa1e5 2024-12-03T14:58:46,371 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNext/ae65c8dcc6508897b56b3fa8382fa1e5 2024-12-03T14:58:46,372 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for ae65c8dcc6508897b56b3fa8382fa1e5 2024-12-03T14:58:46,372 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for ae65c8dcc6508897b56b3fa8382fa1e5 2024-12-03T14:58:46,374 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for ae65c8dcc6508897b56b3fa8382fa1e5 2024-12-03T14:58:46,377 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNext/ae65c8dcc6508897b56b3fa8382fa1e5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:58:46,377 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened ae65c8dcc6508897b56b3fa8382fa1e5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73531578, jitterRate=0.09570589661598206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:58:46,378 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae65c8dcc6508897b56b3fa8382fa1e5 2024-12-03T14:58:46,378 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for ae65c8dcc6508897b56b3fa8382fa1e5: Running coprocessor pre-open hook at 1733237926365Writing region info on filesystem at 1733237926365Initializing all the Stores at 1733237926367 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237926367Cleaning up temporary data from old regions at 1733237926372 (+5 ms)Running coprocessor post-open hooks at 1733237926378 (+6 ms)Region opened successfully at 1733237926378 2024-12-03T14:58:46,379 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5., pid=30, masterSystemTime=1733237926359 2024-12-03T14:58:46,382 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. 2024-12-03T14:58:46,382 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. 2024-12-03T14:58:46,383 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=ae65c8dcc6508897b56b3fa8382fa1e5, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:46,386 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae65c8dcc6508897b56b3fa8382fa1e5, server=a5d22df9eca2,45227,1733237844286 because future has completed 2024-12-03T14:58:46,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-03T14:58:46,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure ae65c8dcc6508897b56b3fa8382fa1e5, server=a5d22df9eca2,45227,1733237844286 in 181 msec 2024-12-03T14:58:46,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-03T14:58:46,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=ae65c8dcc6508897b56b3fa8382fa1e5, ASSIGN in 345 msec 2024-12-03T14:58:46,395 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:58:46,395 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237926395"}]},"ts":"1733237926395"} 2024-12-03T14:58:46,397 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-03T14:58:46,398 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:58:46,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 790 msec 2024-12-03T14:58:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T14:58:46,621 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-03T14:58:46,621 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-03T14:58:46,621 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:46,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-03T14:58:46,627 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:46,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-03T14:58:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2] 2024-12-03T14:58:46,643 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T14:58:46,643 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-03T14:58:46,647 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T14:58:46,650 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T14:58:46,652 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T14:58:46,652 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-03T14:58:50,957 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-03T14:58:51,867 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:58:52,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45227 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6423078111759302604' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T14:58:52,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45227 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:53142 deadline: 1733237938660, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6423078111759302604' does not exist 2024-12-03T14:58:52,663 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to a5d22df9eca2,45227,1733237844286 for closing scanner id = -6423078111759302604 for ae65c8dcc6508897b56b3fa8382fa1e5 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6423078111759302604' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:58:52,668 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=302 (was 284) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4972 (was 4992) 2024-12-03T14:58:52,679 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=302, ProcessCount=11, AvailableMemoryMB=4972 2024-12-03T14:58:52,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:58:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-03T14:58:52,684 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:58:52,684 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:52,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-03T14:58:52,686 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:58:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T14:58:52,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741853_1029 (size=423) 2024-12-03T14:58:52,799 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T14:58:53,097 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0ae91869bd7e27b5706100b506da3795, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e 2024-12-03T14:58:53,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741854_1030 (size=74) 2024-12-03T14:58:53,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-12-03T14:58:53,507 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:53,507 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 0ae91869bd7e27b5706100b506da3795, disabling compactions & flushes 2024-12-03T14:58:53,507 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. 2024-12-03T14:58:53,507 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. 2024-12-03T14:58:53,507 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. after waiting 0 ms 2024-12-03T14:58:53,507 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. 2024-12-03T14:58:53,507 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. 2024-12-03T14:58:53,507 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0ae91869bd7e27b5706100b506da3795: Waiting for close lock at 1733237933507Disabling compacts and flushes for region at 1733237933507Disabling writes for close at 1733237933507Writing region close event to WAL at 1733237933507Closed at 1733237933507 2024-12-03T14:58:53,509 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:58:53,509 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733237933509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237933509"}]},"ts":"1733237933509"} 2024-12-03T14:58:53,512 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:58:53,513 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:58:53,514 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237933514"}]},"ts":"1733237933514"} 2024-12-03T14:58:53,517 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-03T14:58:53,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0ae91869bd7e27b5706100b506da3795, ASSIGN}] 2024-12-03T14:58:53,519 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0ae91869bd7e27b5706100b506da3795, ASSIGN 2024-12-03T14:58:53,520 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0ae91869bd7e27b5706100b506da3795, ASSIGN; state=OFFLINE, location=a5d22df9eca2,45227,1733237844286; forceNewPlan=false, retain=false 2024-12-03T14:58:53,671 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=0ae91869bd7e27b5706100b506da3795, regionState=OPENING, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:53,674 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0ae91869bd7e27b5706100b506da3795, ASSIGN because future has completed 2024-12-03T14:58:53,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ae91869bd7e27b5706100b506da3795, server=a5d22df9eca2,45227,1733237844286}] 2024-12-03T14:58:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T14:58:53,832 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. 2024-12-03T14:58:53,832 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ae91869bd7e27b5706100b506da3795, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:58:53,833 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 0ae91869bd7e27b5706100b506da3795 2024-12-03T14:58:53,833 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:58:53,833 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 0ae91869bd7e27b5706100b506da3795 2024-12-03T14:58:53,833 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 0ae91869bd7e27b5706100b506da3795 2024-12-03T14:58:53,835 INFO [StoreOpener-0ae91869bd7e27b5706100b506da3795-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0ae91869bd7e27b5706100b506da3795 2024-12-03T14:58:53,837 INFO [StoreOpener-0ae91869bd7e27b5706100b506da3795-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ae91869bd7e27b5706100b506da3795 columnFamilyName testFamily 2024-12-03T14:58:53,837 DEBUG [StoreOpener-0ae91869bd7e27b5706100b506da3795-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:58:53,838 INFO [StoreOpener-0ae91869bd7e27b5706100b506da3795-1 {}] regionserver.HStore(327): Store=0ae91869bd7e27b5706100b506da3795/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:58:53,838 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 0ae91869bd7e27b5706100b506da3795 2024-12-03T14:58:53,839 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ae91869bd7e27b5706100b506da3795 2024-12-03T14:58:53,840 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ae91869bd7e27b5706100b506da3795 2024-12-03T14:58:53,840 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 0ae91869bd7e27b5706100b506da3795 2024-12-03T14:58:53,840 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 0ae91869bd7e27b5706100b506da3795 2024-12-03T14:58:53,842 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 0ae91869bd7e27b5706100b506da3795 2024-12-03T14:58:53,845 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ae91869bd7e27b5706100b506da3795/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:58:53,846 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 0ae91869bd7e27b5706100b506da3795; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62246071, jitterRate=-0.07246126234531403}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:58:53,846 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ae91869bd7e27b5706100b506da3795 2024-12-03T14:58:53,847 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 0ae91869bd7e27b5706100b506da3795: Running coprocessor pre-open hook at 1733237933833Writing region info on filesystem at 1733237933833Initializing all the Stores at 1733237933835 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237933835Cleaning up temporary data from old regions at 1733237933840 (+5 ms)Running coprocessor post-open hooks at 1733237933846 (+6 ms)Region opened successfully at 1733237933846 2024-12-03T14:58:53,848 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795., pid=33, masterSystemTime=1733237933827 2024-12-03T14:58:53,850 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. 2024-12-03T14:58:53,850 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. 2024-12-03T14:58:53,851 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=0ae91869bd7e27b5706100b506da3795, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:53,854 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ae91869bd7e27b5706100b506da3795, server=a5d22df9eca2,45227,1733237844286 because future has completed 2024-12-03T14:58:53,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-03T14:58:53,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ae91869bd7e27b5706100b506da3795, server=a5d22df9eca2,45227,1733237844286 in 181 msec 2024-12-03T14:58:53,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-03T14:58:53,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0ae91869bd7e27b5706100b506da3795, ASSIGN in 341 msec 2024-12-03T14:58:53,862 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:58:53,862 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237933862"}]},"ts":"1733237933862"} 2024-12-03T14:58:53,864 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-03T14:58:53,866 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:58:53,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1840 sec 2024-12-03T14:58:54,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-12-03T14:58:54,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T14:58:54,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-03T14:58:54,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-03T14:58:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T14:58:55,711 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-03T14:58:55,711 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-03T14:58:55,711 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:55,715 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-03T14:58:55,716 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:58:55,716 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-03T14:58:55,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2] 2024-12-03T14:58:55,731 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T14:58:55,731 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-03T14:58:55,735 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T14:58:55,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T14:58:56,739 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a5d22df9eca2:45227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-12-03T14:58:56,739 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-12-03T14:58:56,739 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 from cache 2024-12-03T14:58:56,959 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-03T14:58:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-12-03T14:58:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2] 2024-12-03T14:58:57,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45227 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T14:58:58,444 DEBUG [master/a5d22df9eca2:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T14:58:58,739 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a5d22df9eca2:45227 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=981ms, rpcTimeout=977ms 2024-12-03T14:58:58,739 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=981ms, rpcTimeout=977ms 2024-12-03T14:58:58,739 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795., hostname=a5d22df9eca2,45227,1733237844286, seqNum=2 from cache 2024-12-03T14:58:58,747 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=310 (was 302) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4967 (was 4972) 2024-12-03T14:58:58,747 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T14:58:58,747 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:58:58,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:58:58,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:58:58,752 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:58:58,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T14:58:58,752 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T14:58:58,752 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:58:58,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:58:58,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:58:58,752 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:58:58,753 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T14:58:58,753 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1566887813, stopped=false 2024-12-03T14:58:58,753 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a5d22df9eca2,44239,1733237843695 2024-12-03T14:58:58,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T14:58:58,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T14:58:58,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:58:58,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:58:58,755 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T14:58:58,755 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T14:58:58,755 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:58:58,756 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:58:58,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:58:58,756 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:58:58,756 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a5d22df9eca2,45227,1733237844286' ***** 2024-12-03T14:58:58,756 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T14:58:58,756 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T14:58:58,756 INFO [RS:0;a5d22df9eca2:45227 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T14:58:58,757 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T14:58:58,757 INFO [RS:0;a5d22df9eca2:45227 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T14:58:58,757 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(3091): Received CLOSE for b41aac3f4066a49ef9332272456c1a99 2024-12-03T14:58:58,757 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(3091): Received CLOSE for ae65c8dcc6508897b56b3fa8382fa1e5 2024-12-03T14:58:58,757 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(3091): Received CLOSE for 0ae91869bd7e27b5706100b506da3795 2024-12-03T14:58:58,757 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(3091): Received CLOSE for 4776bec4819b23bad8df61b64e7c38b7 2024-12-03T14:58:58,757 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(3091): Received CLOSE for 37ac058b85763227b939db7883e3d341 2024-12-03T14:58:58,757 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(3091): Received CLOSE for 03bbcd680c9a9257b7bf3639be6cfe7a 2024-12-03T14:58:58,757 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(3091): Received CLOSE for ccf86b0ce002ced573170bfc41cbf8bf 2024-12-03T14:58:58,758 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(3091): Received CLOSE for e50ad728224bb2567237109c0c01214d 2024-12-03T14:58:58,758 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(3091): Received CLOSE for ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:58,758 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(3091): Received CLOSE for 57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:58:58,758 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b41aac3f4066a49ef9332272456c1a99, disabling compactions & flushes 2024-12-03T14:58:58,758 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. 2024-12-03T14:58:58,758 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(959): stopping server a5d22df9eca2,45227,1733237844286 2024-12-03T14:58:58,758 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. 2024-12-03T14:58:58,758 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T14:58:58,758 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. after waiting 0 ms 2024-12-03T14:58:58,758 INFO [RS:0;a5d22df9eca2:45227 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a5d22df9eca2:45227. 2024-12-03T14:58:58,758 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. 2024-12-03T14:58:58,758 DEBUG [RS:0;a5d22df9eca2:45227 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:58:58,758 DEBUG [RS:0;a5d22df9eca2:45227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:58:58,758 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T14:58:58,758 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T14:58:58,758 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T14:58:58,758 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T14:58:58,759 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-03T14:58:58,759 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T14:58:58,759 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T14:58:58,759 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T14:58:58,759 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T14:58:58,759 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T14:58:58,759 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 03bbcd680c9a9257b7bf3639be6cfe7a, 0ae91869bd7e27b5706100b506da3795, 1588230740, 37ac058b85763227b939db7883e3d341, 4776bec4819b23bad8df61b64e7c38b7, 57c89a40077913e3c2dae3b7512725f9, ae65c8dcc6508897b56b3fa8382fa1e5, b41aac3f4066a49ef9332272456c1a99, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:58,761 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-03T14:58:58,761 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b41aac3f4066a49ef9332272456c1a99 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T14:58:58,813 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScanner/b41aac3f4066a49ef9332272456c1a99/.tmp/testFamily/cf0baee52db94fccbccd745c7735e2a0 is 61, key is row-0/testFamily:testQualifier/1733237901260/Put/seqid=0 2024-12-03T14:58:58,814 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/.tmp/info/ef564a0b0e134b768a6a7cbbc89b6c1d is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341./info:regioninfo/1733237878957/Put/seqid=0 2024-12-03T14:58:58,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741855_1031 (size=20564) 2024-12-03T14:58:58,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741856_1032 (size=5049) 2024-12-03T14:58:58,914 INFO [regionserver/a5d22df9eca2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T14:58:58,914 INFO [regionserver/a5d22df9eca2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T14:58:58,959 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 03bbcd680c9a9257b7bf3639be6cfe7a, 0ae91869bd7e27b5706100b506da3795, 1588230740, 37ac058b85763227b939db7883e3d341, 4776bec4819b23bad8df61b64e7c38b7, 57c89a40077913e3c2dae3b7512725f9, ae65c8dcc6508897b56b3fa8382fa1e5, b41aac3f4066a49ef9332272456c1a99, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:59,160 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 03bbcd680c9a9257b7bf3639be6cfe7a, 0ae91869bd7e27b5706100b506da3795, 1588230740, 37ac058b85763227b939db7883e3d341, 4776bec4819b23bad8df61b64e7c38b7, 57c89a40077913e3c2dae3b7512725f9, ae65c8dcc6508897b56b3fa8382fa1e5, b41aac3f4066a49ef9332272456c1a99, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:59,227 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/.tmp/info/ef564a0b0e134b768a6a7cbbc89b6c1d 2024-12-03T14:58:59,227 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScanner/b41aac3f4066a49ef9332272456c1a99/.tmp/testFamily/cf0baee52db94fccbccd745c7735e2a0 2024-12-03T14:58:59,259 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-03T14:58:59,271 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScanner/b41aac3f4066a49ef9332272456c1a99/.tmp/testFamily/cf0baee52db94fccbccd745c7735e2a0 as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScanner/b41aac3f4066a49ef9332272456c1a99/testFamily/cf0baee52db94fccbccd745c7735e2a0 2024-12-03T14:58:59,279 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScanner/b41aac3f4066a49ef9332272456c1a99/testFamily/cf0baee52db94fccbccd745c7735e2a0, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T14:58:59,285 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/.tmp/ns/64b052c1efeb49a2addf138f23e546ac is 43, key is default/ns:d/1733237877492/Put/seqid=0 2024-12-03T14:58:59,288 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b41aac3f4066a49ef9332272456c1a99 in 526ms, sequenceid=8, compaction requested=false 2024-12-03T14:58:59,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741857_1033 (size=5153) 2024-12-03T14:58:59,293 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScanner/b41aac3f4066a49ef9332272456c1a99/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T14:58:59,296 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. 2024-12-03T14:58:59,296 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b41aac3f4066a49ef9332272456c1a99: Waiting for close lock at 1733237938757Running coprocessor pre-close hooks at 1733237938758 (+1 ms)Disabling compacts and flushes for region at 1733237938758Disabling writes for close at 1733237938758Obtaining lock to block concurrent updates at 1733237938761 (+3 ms)Preparing flush snapshotting stores in b41aac3f4066a49ef9332272456c1a99 at 1733237938761Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733237938767 (+6 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. at 1733237938769 (+2 ms)Flushing b41aac3f4066a49ef9332272456c1a99/testFamily: creating writer at 1733237938770 (+1 ms)Flushing b41aac3f4066a49ef9332272456c1a99/testFamily: appending metadata at 1733237938808 (+38 ms)Flushing b41aac3f4066a49ef9332272456c1a99/testFamily: closing flushed file at 1733237938811 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ba1e856: reopening flushed file at 1733237939269 (+458 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b41aac3f4066a49ef9332272456c1a99 in 526ms, sequenceid=8, compaction requested=false at 1733237939288 (+19 ms)Writing region close event to WAL at 1733237939289 (+1 ms)Running coprocessor post-close hooks at 1733237939294 (+5 ms)Closed at 1733237939296 (+2 ms) 2024-12-03T14:58:59,296 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733237900223.b41aac3f4066a49ef9332272456c1a99. 2024-12-03T14:58:59,296 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ae65c8dcc6508897b56b3fa8382fa1e5, disabling compactions & flushes 2024-12-03T14:58:59,296 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. 2024-12-03T14:58:59,296 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. 2024-12-03T14:58:59,296 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. after waiting 0 ms 2024-12-03T14:58:59,296 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. 2024-12-03T14:58:59,297 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ae65c8dcc6508897b56b3fa8382fa1e5 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T14:58:59,311 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNext/ae65c8dcc6508897b56b3fa8382fa1e5/.tmp/testFamily/1d4a50ecd3f342309288a67649821a68 is 61, key is row-0/testFamily:testQualifier/1733237926632/Put/seqid=0 2024-12-03T14:58:59,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741858_1034 (size=5049) 2024-12-03T14:58:59,334 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:58:59,360 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 03bbcd680c9a9257b7bf3639be6cfe7a, 0ae91869bd7e27b5706100b506da3795, 1588230740, 37ac058b85763227b939db7883e3d341, 4776bec4819b23bad8df61b64e7c38b7, 57c89a40077913e3c2dae3b7512725f9, ae65c8dcc6508897b56b3fa8382fa1e5, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:59,561 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 03bbcd680c9a9257b7bf3639be6cfe7a, 0ae91869bd7e27b5706100b506da3795, 1588230740, 37ac058b85763227b939db7883e3d341, 4776bec4819b23bad8df61b64e7c38b7, 57c89a40077913e3c2dae3b7512725f9, ae65c8dcc6508897b56b3fa8382fa1e5, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:59,692 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/.tmp/ns/64b052c1efeb49a2addf138f23e546ac 2024-12-03T14:58:59,716 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNext/ae65c8dcc6508897b56b3fa8382fa1e5/.tmp/testFamily/1d4a50ecd3f342309288a67649821a68 2024-12-03T14:58:59,726 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNext/ae65c8dcc6508897b56b3fa8382fa1e5/.tmp/testFamily/1d4a50ecd3f342309288a67649821a68 as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNext/ae65c8dcc6508897b56b3fa8382fa1e5/testFamily/1d4a50ecd3f342309288a67649821a68 2024-12-03T14:58:59,727 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/.tmp/table/e403e2d7ffc34140809819a01ca9cfb2 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733237878973/Put/seqid=0 2024-12-03T14:58:59,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741859_1035 (size=6699) 2024-12-03T14:58:59,732 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/.tmp/table/e403e2d7ffc34140809819a01ca9cfb2 2024-12-03T14:58:59,734 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNext/ae65c8dcc6508897b56b3fa8382fa1e5/testFamily/1d4a50ecd3f342309288a67649821a68, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T14:58:59,735 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ae65c8dcc6508897b56b3fa8382fa1e5 in 439ms, sequenceid=8, compaction requested=false 2024-12-03T14:58:59,740 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNext/ae65c8dcc6508897b56b3fa8382fa1e5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T14:58:59,742 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. 2024-12-03T14:58:59,742 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ae65c8dcc6508897b56b3fa8382fa1e5: Waiting for close lock at 1733237939296Running coprocessor pre-close hooks at 1733237939296Disabling compacts and flushes for region at 1733237939296Disabling writes for close at 1733237939296Obtaining lock to block concurrent updates at 1733237939297 (+1 ms)Preparing flush snapshotting stores in ae65c8dcc6508897b56b3fa8382fa1e5 at 1733237939297Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733237939297Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. at 1733237939297Flushing ae65c8dcc6508897b56b3fa8382fa1e5/testFamily: creating writer at 1733237939298 (+1 ms)Flushing ae65c8dcc6508897b56b3fa8382fa1e5/testFamily: appending metadata at 1733237939311 (+13 ms)Flushing ae65c8dcc6508897b56b3fa8382fa1e5/testFamily: closing flushed file at 1733237939311Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f3a6e50: reopening flushed file at 1733237939725 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ae65c8dcc6508897b56b3fa8382fa1e5 in 439ms, sequenceid=8, compaction requested=false at 1733237939735 (+10 ms)Writing region close event to WAL at 1733237939736 (+1 ms)Running coprocessor post-close hooks at 1733237939741 (+5 ms)Closed at 1733237939741 2024-12-03T14:58:59,742 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733237925608.ae65c8dcc6508897b56b3fa8382fa1e5. 2024-12-03T14:58:59,742 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0ae91869bd7e27b5706100b506da3795, disabling compactions & flushes 2024-12-03T14:58:59,742 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. 2024-12-03T14:58:59,742 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. 2024-12-03T14:58:59,742 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/.tmp/info/ef564a0b0e134b768a6a7cbbc89b6c1d as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/info/ef564a0b0e134b768a6a7cbbc89b6c1d 2024-12-03T14:58:59,742 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. after waiting 0 ms 2024-12-03T14:58:59,742 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. 2024-12-03T14:58:59,742 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0ae91869bd7e27b5706100b506da3795 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T14:58:59,753 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/info/ef564a0b0e134b768a6a7cbbc89b6c1d, entries=100, sequenceid=56, filesize=20.1 K 2024-12-03T14:58:59,755 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/.tmp/ns/64b052c1efeb49a2addf138f23e546ac as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/ns/64b052c1efeb49a2addf138f23e546ac 2024-12-03T14:58:59,761 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-12-03T14:58:59,761 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1325): Online Regions={0ae91869bd7e27b5706100b506da3795=testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795., 4776bec4819b23bad8df61b64e7c38b7=testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7., 37ac058b85763227b939db7883e3d341=testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341., 03bbcd680c9a9257b7bf3639be6cfe7a=testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a., ccf86b0ce002ced573170bfc41cbf8bf=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf., 1588230740=hbase:meta,,1.1588230740, e50ad728224bb2567237109c0c01214d=hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d., ef7b07aadb8991db2c582b540fe7b844=hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844., 57c89a40077913e3c2dae3b7512725f9=testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9.} 2024-12-03T14:58:59,761 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 03bbcd680c9a9257b7bf3639be6cfe7a, 0ae91869bd7e27b5706100b506da3795, 1588230740, 37ac058b85763227b939db7883e3d341, 4776bec4819b23bad8df61b64e7c38b7, 57c89a40077913e3c2dae3b7512725f9, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:58:59,763 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ae91869bd7e27b5706100b506da3795/.tmp/testFamily/2a234f6c9ecb4605a9847b0531a12276 is 61, key is row-0/testFamily:testQualifier/1733237935721/Put/seqid=0 2024-12-03T14:58:59,765 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/ns/64b052c1efeb49a2addf138f23e546ac, entries=2, sequenceid=56, filesize=5.0 K 2024-12-03T14:58:59,766 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/.tmp/table/e403e2d7ffc34140809819a01ca9cfb2 as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/table/e403e2d7ffc34140809819a01ca9cfb2 2024-12-03T14:58:59,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741860_1036 (size=5049) 2024-12-03T14:58:59,777 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/table/e403e2d7ffc34140809819a01ca9cfb2, entries=20, sequenceid=56, filesize=6.5 K 2024-12-03T14:58:59,778 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1019ms, sequenceid=56, compaction requested=false 2024-12-03T14:58:59,784 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-03T14:58:59,785 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T14:58:59,786 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T14:58:59,786 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733237938759Running coprocessor pre-close hooks at 1733237938759Disabling compacts and flushes for region at 1733237938759Disabling writes for close at 1733237938759Obtaining lock to block concurrent updates at 1733237938761 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1733237938761Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733237938767 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733237938769 (+2 ms)Flushing 1588230740/info: creating writer at 1733237938770 (+1 ms)Flushing 1588230740/info: appending metadata at 1733237938811 (+41 ms)Flushing 1588230740/info: closing flushed file at 1733237938811Flushing 1588230740/ns: creating writer at 1733237939269 (+458 ms)Flushing 1588230740/ns: appending metadata at 1733237939284 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733237939284Flushing 1588230740/table: creating writer at 1733237939707 (+423 ms)Flushing 1588230740/table: appending metadata at 1733237939726 (+19 ms)Flushing 1588230740/table: closing flushed file at 1733237939726Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29ed1364: reopening flushed file at 1733237939741 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52325148: reopening flushed file at 1733237939754 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52dd6420: reopening flushed file at 1733237939765 (+11 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1019ms, sequenceid=56, compaction requested=false at 1733237939778 (+13 ms)Writing region close event to WAL at 1733237939779 (+1 ms)Running coprocessor post-close hooks at 1733237939785 (+6 ms)Closed at 1733237939786 (+1 ms) 2024-12-03T14:58:59,786 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T14:58:59,961 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 03bbcd680c9a9257b7bf3639be6cfe7a, 0ae91869bd7e27b5706100b506da3795, 37ac058b85763227b939db7883e3d341, 4776bec4819b23bad8df61b64e7c38b7, 57c89a40077913e3c2dae3b7512725f9, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:00,162 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 03bbcd680c9a9257b7bf3639be6cfe7a, 0ae91869bd7e27b5706100b506da3795, 37ac058b85763227b939db7883e3d341, 4776bec4819b23bad8df61b64e7c38b7, 57c89a40077913e3c2dae3b7512725f9, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:00,171 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ae91869bd7e27b5706100b506da3795/.tmp/testFamily/2a234f6c9ecb4605a9847b0531a12276 2024-12-03T14:59:00,185 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ae91869bd7e27b5706100b506da3795/.tmp/testFamily/2a234f6c9ecb4605a9847b0531a12276 as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ae91869bd7e27b5706100b506da3795/testFamily/2a234f6c9ecb4605a9847b0531a12276 2024-12-03T14:59:00,193 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ae91869bd7e27b5706100b506da3795/testFamily/2a234f6c9ecb4605a9847b0531a12276, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T14:59:00,194 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0ae91869bd7e27b5706100b506da3795 in 452ms, sequenceid=8, compaction requested=false 2024-12-03T14:59:00,199 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnOpenScannerAsync/0ae91869bd7e27b5706100b506da3795/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T14:59:00,200 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. 2024-12-03T14:59:00,200 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0ae91869bd7e27b5706100b506da3795: Waiting for close lock at 1733237939742Running coprocessor pre-close hooks at 1733237939742Disabling compacts and flushes for region at 1733237939742Disabling writes for close at 1733237939742Obtaining lock to block concurrent updates at 1733237939742Preparing flush snapshotting stores in 0ae91869bd7e27b5706100b506da3795 at 1733237939742Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733237939743 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. at 1733237939744 (+1 ms)Flushing 0ae91869bd7e27b5706100b506da3795/testFamily: creating writer at 1733237939744Flushing 0ae91869bd7e27b5706100b506da3795/testFamily: appending metadata at 1733237939762 (+18 ms)Flushing 0ae91869bd7e27b5706100b506da3795/testFamily: closing flushed file at 1733237939762Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3acfef3e: reopening flushed file at 1733237940184 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0ae91869bd7e27b5706100b506da3795 in 452ms, sequenceid=8, compaction requested=false at 1733237940194 (+10 ms)Writing region close event to WAL at 1733237940195 (+1 ms)Running coprocessor post-close hooks at 1733237940200 (+5 ms)Closed at 1733237940200 2024-12-03T14:59:00,201 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733237932681.0ae91869bd7e27b5706100b506da3795. 2024-12-03T14:59:00,201 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4776bec4819b23bad8df61b64e7c38b7, disabling compactions & flushes 2024-12-03T14:59:00,201 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. 2024-12-03T14:59:00,201 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. 2024-12-03T14:59:00,201 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. after waiting 0 ms 2024-12-03T14:59:00,201 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. 2024-12-03T14:59:00,201 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4776bec4819b23bad8df61b64e7c38b7 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T14:59:00,220 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNext/4776bec4819b23bad8df61b64e7c38b7/.tmp/testFamily/f5d5ea9d894c4a0ab54faec6a826aad6 is 61, key is row-0/testFamily:testQualifier/1733237913479/Put/seqid=0 2024-12-03T14:59:00,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741861_1037 (size=5049) 2024-12-03T14:59:00,362 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 03bbcd680c9a9257b7bf3639be6cfe7a, 37ac058b85763227b939db7883e3d341, 4776bec4819b23bad8df61b64e7c38b7, 57c89a40077913e3c2dae3b7512725f9, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:00,563 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 03bbcd680c9a9257b7bf3639be6cfe7a, 37ac058b85763227b939db7883e3d341, 4776bec4819b23bad8df61b64e7c38b7, 57c89a40077913e3c2dae3b7512725f9, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:00,627 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNext/4776bec4819b23bad8df61b64e7c38b7/.tmp/testFamily/f5d5ea9d894c4a0ab54faec6a826aad6 2024-12-03T14:59:00,642 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNext/4776bec4819b23bad8df61b64e7c38b7/.tmp/testFamily/f5d5ea9d894c4a0ab54faec6a826aad6 as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNext/4776bec4819b23bad8df61b64e7c38b7/testFamily/f5d5ea9d894c4a0ab54faec6a826aad6 2024-12-03T14:59:00,648 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNext/4776bec4819b23bad8df61b64e7c38b7/testFamily/f5d5ea9d894c4a0ab54faec6a826aad6, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T14:59:00,649 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4776bec4819b23bad8df61b64e7c38b7 in 448ms, sequenceid=8, compaction requested=false 2024-12-03T14:59:00,654 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNext/4776bec4819b23bad8df61b64e7c38b7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T14:59:00,655 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. 2024-12-03T14:59:00,655 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4776bec4819b23bad8df61b64e7c38b7: Waiting for close lock at 1733237940201Running coprocessor pre-close hooks at 1733237940201Disabling compacts and flushes for region at 1733237940201Disabling writes for close at 1733237940201Obtaining lock to block concurrent updates at 1733237940201Preparing flush snapshotting stores in 4776bec4819b23bad8df61b64e7c38b7 at 1733237940201Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733237940202 (+1 ms)Flushing stores of testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. at 1733237940203 (+1 ms)Flushing 4776bec4819b23bad8df61b64e7c38b7/testFamily: creating writer at 1733237940203Flushing 4776bec4819b23bad8df61b64e7c38b7/testFamily: appending metadata at 1733237940219 (+16 ms)Flushing 4776bec4819b23bad8df61b64e7c38b7/testFamily: closing flushed file at 1733237940220 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27f9d14d: reopening flushed file at 1733237940640 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4776bec4819b23bad8df61b64e7c38b7 in 448ms, sequenceid=8, compaction requested=false at 1733237940649 (+9 ms)Writing region close event to WAL at 1733237940650 (+1 ms)Running coprocessor post-close hooks at 1733237940655 (+5 ms)Closed at 1733237940655 2024-12-03T14:59:00,655 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733237910418.4776bec4819b23bad8df61b64e7c38b7. 2024-12-03T14:59:00,655 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 37ac058b85763227b939db7883e3d341, disabling compactions & flushes 2024-12-03T14:59:00,655 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. 2024-12-03T14:59:00,655 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. 2024-12-03T14:59:00,656 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. after waiting 0 ms 2024-12-03T14:59:00,656 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. 2024-12-03T14:59:00,656 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 37ac058b85763227b939db7883e3d341 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T14:59:00,672 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/37ac058b85763227b939db7883e3d341/.tmp/testFamily/c6682947faad4f308d42dac2615cbae9 is 61, key is row-0/testFamily:testQualifier/1733237880812/Put/seqid=0 2024-12-03T14:59:00,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741862_1038 (size=5049) 2024-12-03T14:59:00,763 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-12-03T14:59:00,763 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1325): Online Regions={37ac058b85763227b939db7883e3d341=testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341., 03bbcd680c9a9257b7bf3639be6cfe7a=testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a., ccf86b0ce002ced573170bfc41cbf8bf=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf., e50ad728224bb2567237109c0c01214d=hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d., ef7b07aadb8991db2c582b540fe7b844=hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844., 57c89a40077913e3c2dae3b7512725f9=testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9.} 2024-12-03T14:59:00,763 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 03bbcd680c9a9257b7bf3639be6cfe7a, 37ac058b85763227b939db7883e3d341, 57c89a40077913e3c2dae3b7512725f9, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:00,964 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 03bbcd680c9a9257b7bf3639be6cfe7a, 37ac058b85763227b939db7883e3d341, 57c89a40077913e3c2dae3b7512725f9, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:01,079 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/37ac058b85763227b939db7883e3d341/.tmp/testFamily/c6682947faad4f308d42dac2615cbae9 2024-12-03T14:59:01,094 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/37ac058b85763227b939db7883e3d341/.tmp/testFamily/c6682947faad4f308d42dac2615cbae9 as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/37ac058b85763227b939db7883e3d341/testFamily/c6682947faad4f308d42dac2615cbae9 2024-12-03T14:59:01,103 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/37ac058b85763227b939db7883e3d341/testFamily/c6682947faad4f308d42dac2615cbae9, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T14:59:01,105 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 37ac058b85763227b939db7883e3d341 in 449ms, sequenceid=8, compaction requested=false 2024-12-03T14:59:01,111 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/37ac058b85763227b939db7883e3d341/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T14:59:01,111 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. 2024-12-03T14:59:01,112 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 37ac058b85763227b939db7883e3d341: Waiting for close lock at 1733237940655Running coprocessor pre-close hooks at 1733237940655Disabling compacts and flushes for region at 1733237940655Disabling writes for close at 1733237940656 (+1 ms)Obtaining lock to block concurrent updates at 1733237940656Preparing flush snapshotting stores in 37ac058b85763227b939db7883e3d341 at 1733237940656Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733237940656Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. at 1733237940657 (+1 ms)Flushing 37ac058b85763227b939db7883e3d341/testFamily: creating writer at 1733237940657Flushing 37ac058b85763227b939db7883e3d341/testFamily: appending metadata at 1733237940671 (+14 ms)Flushing 37ac058b85763227b939db7883e3d341/testFamily: closing flushed file at 1733237940671Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@472f7d61: reopening flushed file at 1733237941093 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 37ac058b85763227b939db7883e3d341 in 449ms, sequenceid=8, compaction requested=false at 1733237941105 (+12 ms)Writing region close event to WAL at 1733237941106 (+1 ms)Running coprocessor post-close hooks at 1733237941111 (+5 ms)Closed at 1733237941111 2024-12-03T14:59:01,112 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733237877701.37ac058b85763227b939db7883e3d341. 2024-12-03T14:59:01,112 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 03bbcd680c9a9257b7bf3639be6cfe7a, disabling compactions & flushes 2024-12-03T14:59:01,112 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. 2024-12-03T14:59:01,112 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. 2024-12-03T14:59:01,112 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. after waiting 0 ms 2024-12-03T14:59:01,112 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. 2024-12-03T14:59:01,112 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 03bbcd680c9a9257b7bf3639be6cfe7a 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T14:59:01,127 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNextAsync/03bbcd680c9a9257b7bf3639be6cfe7a/.tmp/testFamily/4d7a53f4737b49849696be317e330b3e is 61, key is row-0/testFamily:testQualifier/1733237907373/Put/seqid=0 2024-12-03T14:59:01,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741863_1039 (size=5049) 2024-12-03T14:59:01,164 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 03bbcd680c9a9257b7bf3639be6cfe7a, 57c89a40077913e3c2dae3b7512725f9, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:01,364 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 03bbcd680c9a9257b7bf3639be6cfe7a, 57c89a40077913e3c2dae3b7512725f9, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:01,535 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNextAsync/03bbcd680c9a9257b7bf3639be6cfe7a/.tmp/testFamily/4d7a53f4737b49849696be317e330b3e 2024-12-03T14:59:01,550 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNextAsync/03bbcd680c9a9257b7bf3639be6cfe7a/.tmp/testFamily/4d7a53f4737b49849696be317e330b3e as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNextAsync/03bbcd680c9a9257b7bf3639be6cfe7a/testFamily/4d7a53f4737b49849696be317e330b3e 2024-12-03T14:59:01,558 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNextAsync/03bbcd680c9a9257b7bf3639be6cfe7a/testFamily/4d7a53f4737b49849696be317e330b3e, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T14:59:01,559 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 03bbcd680c9a9257b7bf3639be6cfe7a in 446ms, sequenceid=8, compaction requested=false 2024-12-03T14:59:01,563 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testNormalScanTimeoutOnNextAsync/03bbcd680c9a9257b7bf3639be6cfe7a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T14:59:01,563 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. 2024-12-03T14:59:01,563 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 03bbcd680c9a9257b7bf3639be6cfe7a: Waiting for close lock at 1733237941112Running coprocessor pre-close hooks at 1733237941112Disabling compacts and flushes for region at 1733237941112Disabling writes for close at 1733237941112Obtaining lock to block concurrent updates at 1733237941112Preparing flush snapshotting stores in 03bbcd680c9a9257b7bf3639be6cfe7a at 1733237941112Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733237941113 (+1 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. at 1733237941113Flushing 03bbcd680c9a9257b7bf3639be6cfe7a/testFamily: creating writer at 1733237941113Flushing 03bbcd680c9a9257b7bf3639be6cfe7a/testFamily: appending metadata at 1733237941127 (+14 ms)Flushing 03bbcd680c9a9257b7bf3639be6cfe7a/testFamily: closing flushed file at 1733237941127Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38f2fe25: reopening flushed file at 1733237941548 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 03bbcd680c9a9257b7bf3639be6cfe7a in 446ms, sequenceid=8, compaction requested=false at 1733237941559 (+11 ms)Writing region close event to WAL at 1733237941559Running coprocessor post-close hooks at 1733237941563 (+4 ms)Closed at 1733237941563 2024-12-03T14:59:01,564 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733237904309.03bbcd680c9a9257b7bf3639be6cfe7a. 2024-12-03T14:59:01,564 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ccf86b0ce002ced573170bfc41cbf8bf, disabling compactions & flushes 2024-12-03T14:59:01,564 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. 2024-12-03T14:59:01,564 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. 2024-12-03T14:59:01,564 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. after waiting 0 ms 2024-12-03T14:59:01,564 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. 2024-12-03T14:59:01,564 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ccf86b0ce002ced573170bfc41cbf8bf 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T14:59:01,565 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 57c89a40077913e3c2dae3b7512725f9, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:01,579 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/ccf86b0ce002ced573170bfc41cbf8bf/.tmp/testFamily/1c38b2507a4c4c5e9844402b02fee431 is 61, key is row-0/testFamily:testQualifier/1733237919571/Put/seqid=0 2024-12-03T14:59:01,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741864_1040 (size=5049) 2024-12-03T14:59:01,765 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-12-03T14:59:01,765 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1325): Online Regions={ccf86b0ce002ced573170bfc41cbf8bf=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf., e50ad728224bb2567237109c0c01214d=hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d., ef7b07aadb8991db2c582b540fe7b844=hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844., 57c89a40077913e3c2dae3b7512725f9=testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9.} 2024-12-03T14:59:01,765 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 57c89a40077913e3c2dae3b7512725f9, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:01,966 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 57c89a40077913e3c2dae3b7512725f9, ccf86b0ce002ced573170bfc41cbf8bf, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:01,985 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/ccf86b0ce002ced573170bfc41cbf8bf/.tmp/testFamily/1c38b2507a4c4c5e9844402b02fee431 2024-12-03T14:59:02,002 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/ccf86b0ce002ced573170bfc41cbf8bf/.tmp/testFamily/1c38b2507a4c4c5e9844402b02fee431 as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/ccf86b0ce002ced573170bfc41cbf8bf/testFamily/1c38b2507a4c4c5e9844402b02fee431 2024-12-03T14:59:02,010 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/ccf86b0ce002ced573170bfc41cbf8bf/testFamily/1c38b2507a4c4c5e9844402b02fee431, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T14:59:02,011 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ccf86b0ce002ced573170bfc41cbf8bf in 447ms, sequenceid=8, compaction requested=false 2024-12-03T14:59:02,017 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/ccf86b0ce002ced573170bfc41cbf8bf/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T14:59:02,018 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. 2024-12-03T14:59:02,018 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ccf86b0ce002ced573170bfc41cbf8bf: Waiting for close lock at 1733237941564Running coprocessor pre-close hooks at 1733237941564Disabling compacts and flushes for region at 1733237941564Disabling writes for close at 1733237941564Obtaining lock to block concurrent updates at 1733237941564Preparing flush snapshotting stores in ccf86b0ce002ced573170bfc41cbf8bf at 1733237941564Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733237941564Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. at 1733237941565 (+1 ms)Flushing ccf86b0ce002ced573170bfc41cbf8bf/testFamily: creating writer at 1733237941565Flushing ccf86b0ce002ced573170bfc41cbf8bf/testFamily: appending metadata at 1733237941579 (+14 ms)Flushing ccf86b0ce002ced573170bfc41cbf8bf/testFamily: closing flushed file at 1733237941579Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@400eecbc: reopening flushed file at 1733237942000 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ccf86b0ce002ced573170bfc41cbf8bf in 447ms, sequenceid=8, compaction requested=false at 1733237942011 (+11 ms)Writing region close event to WAL at 1733237942012 (+1 ms)Running coprocessor post-close hooks at 1733237942017 (+5 ms)Closed at 1733237942017 2024-12-03T14:59:02,018 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733237916526.ccf86b0ce002ced573170bfc41cbf8bf. 2024-12-03T14:59:02,018 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e50ad728224bb2567237109c0c01214d, disabling compactions & flushes 2024-12-03T14:59:02,018 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. 2024-12-03T14:59:02,018 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. 2024-12-03T14:59:02,018 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. after waiting 0 ms 2024-12-03T14:59:02,018 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. 2024-12-03T14:59:02,018 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e50ad728224bb2567237109c0c01214d 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T14:59:02,036 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScanner/e50ad728224bb2567237109c0c01214d/.tmp/testFamily/a082d64b766745efb70be964b3dfb7f9 is 61, key is row-0/testFamily:testQualifier/1733237883973/Put/seqid=0 2024-12-03T14:59:02,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741865_1041 (size=5049) 2024-12-03T14:59:02,166 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 57c89a40077913e3c2dae3b7512725f9, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:02,367 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 57c89a40077913e3c2dae3b7512725f9, e50ad728224bb2567237109c0c01214d, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:02,443 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScanner/e50ad728224bb2567237109c0c01214d/.tmp/testFamily/a082d64b766745efb70be964b3dfb7f9 2024-12-03T14:59:02,456 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScanner/e50ad728224bb2567237109c0c01214d/.tmp/testFamily/a082d64b766745efb70be964b3dfb7f9 as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScanner/e50ad728224bb2567237109c0c01214d/testFamily/a082d64b766745efb70be964b3dfb7f9 2024-12-03T14:59:02,465 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScanner/e50ad728224bb2567237109c0c01214d/testFamily/a082d64b766745efb70be964b3dfb7f9, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T14:59:02,466 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e50ad728224bb2567237109c0c01214d in 448ms, sequenceid=8, compaction requested=false 2024-12-03T14:59:02,472 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnOpenScanner/e50ad728224bb2567237109c0c01214d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T14:59:02,473 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. 2024-12-03T14:59:02,473 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e50ad728224bb2567237109c0c01214d: Waiting for close lock at 1733237942018Running coprocessor pre-close hooks at 1733237942018Disabling compacts and flushes for region at 1733237942018Disabling writes for close at 1733237942018Obtaining lock to block concurrent updates at 1733237942018Preparing flush snapshotting stores in e50ad728224bb2567237109c0c01214d at 1733237942018Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733237942019 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. at 1733237942019Flushing e50ad728224bb2567237109c0c01214d/testFamily: creating writer at 1733237942019Flushing e50ad728224bb2567237109c0c01214d/testFamily: appending metadata at 1733237942036 (+17 ms)Flushing e50ad728224bb2567237109c0c01214d/testFamily: closing flushed file at 1733237942036Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f11ebb8: reopening flushed file at 1733237942454 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e50ad728224bb2567237109c0c01214d in 448ms, sequenceid=8, compaction requested=false at 1733237942466 (+12 ms)Writing region close event to WAL at 1733237942467 (+1 ms)Running coprocessor post-close hooks at 1733237942473 (+6 ms)Closed at 1733237942473 2024-12-03T14:59:02,473 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733237880908.e50ad728224bb2567237109c0c01214d. 2024-12-03T14:59:02,473 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ef7b07aadb8991db2c582b540fe7b844, disabling compactions & flushes 2024-12-03T14:59:02,473 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. 2024-12-03T14:59:02,473 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. 2024-12-03T14:59:02,473 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. after waiting 0 ms 2024-12-03T14:59:02,473 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. 2024-12-03T14:59:02,473 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ef7b07aadb8991db2c582b540fe7b844 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T14:59:02,488 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNextAsync/ef7b07aadb8991db2c582b540fe7b844/.tmp/testFamily/4215c33f0a9047be956a8ed6f048df49 is 61, key is row-0/testFamily:testQualifier/1733237891062/Put/seqid=0 2024-12-03T14:59:02,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741866_1042 (size=5049) 2024-12-03T14:59:02,567 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 57c89a40077913e3c2dae3b7512725f9, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:02,767 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-03T14:59:02,768 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1325): Online Regions={ef7b07aadb8991db2c582b540fe7b844=hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844., 57c89a40077913e3c2dae3b7512725f9=testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9.} 2024-12-03T14:59:02,768 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 57c89a40077913e3c2dae3b7512725f9, ef7b07aadb8991db2c582b540fe7b844 2024-12-03T14:59:02,794 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a5d22df9eca2:44239 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 10 more 2024-12-03T14:59:02,895 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNextAsync/ef7b07aadb8991db2c582b540fe7b844/.tmp/testFamily/4215c33f0a9047be956a8ed6f048df49 2024-12-03T14:59:02,908 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNextAsync/ef7b07aadb8991db2c582b540fe7b844/.tmp/testFamily/4215c33f0a9047be956a8ed6f048df49 as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNextAsync/ef7b07aadb8991db2c582b540fe7b844/testFamily/4215c33f0a9047be956a8ed6f048df49 2024-12-03T14:59:02,915 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNextAsync/ef7b07aadb8991db2c582b540fe7b844/testFamily/4215c33f0a9047be956a8ed6f048df49, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T14:59:02,916 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ef7b07aadb8991db2c582b540fe7b844 in 443ms, sequenceid=8, compaction requested=false 2024-12-03T14:59:02,921 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/hbase/testMetaScanTimeoutOnNextAsync/ef7b07aadb8991db2c582b540fe7b844/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T14:59:02,922 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. 2024-12-03T14:59:02,922 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ef7b07aadb8991db2c582b540fe7b844: Waiting for close lock at 1733237942473Running coprocessor pre-close hooks at 1733237942473Disabling compacts and flushes for region at 1733237942473Disabling writes for close at 1733237942473Obtaining lock to block concurrent updates at 1733237942473Preparing flush snapshotting stores in ef7b07aadb8991db2c582b540fe7b844 at 1733237942473Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733237942474 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. at 1733237942474Flushing ef7b07aadb8991db2c582b540fe7b844/testFamily: creating writer at 1733237942474Flushing ef7b07aadb8991db2c582b540fe7b844/testFamily: appending metadata at 1733237942488 (+14 ms)Flushing ef7b07aadb8991db2c582b540fe7b844/testFamily: closing flushed file at 1733237942488Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f7b76fd: reopening flushed file at 1733237942907 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ef7b07aadb8991db2c582b540fe7b844 in 443ms, sequenceid=8, compaction requested=false at 1733237942916 (+9 ms)Writing region close event to WAL at 1733237942917 (+1 ms)Running coprocessor post-close hooks at 1733237942922 (+5 ms)Closed at 1733237942922 2024-12-03T14:59:02,922 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733237890027.ef7b07aadb8991db2c582b540fe7b844. 2024-12-03T14:59:02,922 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 57c89a40077913e3c2dae3b7512725f9, disabling compactions & flushes 2024-12-03T14:59:02,922 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. 2024-12-03T14:59:02,922 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. 2024-12-03T14:59:02,922 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. after waiting 0 ms 2024-12-03T14:59:02,922 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. 2024-12-03T14:59:02,922 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 57c89a40077913e3c2dae3b7512725f9 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T14:59:02,937 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextException/57c89a40077913e3c2dae3b7512725f9/.tmp/testFamily/115e9759b1ed483eb8668a44d6bb65b1 is 61, key is row-0/testFamily:testQualifier/1733237900162/Put/seqid=0 2024-12-03T14:59:02,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741867_1043 (size=5049) 2024-12-03T14:59:02,968 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:59:03,169 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:59:03,344 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextException/57c89a40077913e3c2dae3b7512725f9/.tmp/testFamily/115e9759b1ed483eb8668a44d6bb65b1 2024-12-03T14:59:03,358 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextException/57c89a40077913e3c2dae3b7512725f9/.tmp/testFamily/115e9759b1ed483eb8668a44d6bb65b1 as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextException/57c89a40077913e3c2dae3b7512725f9/testFamily/115e9759b1ed483eb8668a44d6bb65b1 2024-12-03T14:59:03,365 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextException/57c89a40077913e3c2dae3b7512725f9/testFamily/115e9759b1ed483eb8668a44d6bb65b1, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T14:59:03,366 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 57c89a40077913e3c2dae3b7512725f9 in 444ms, sequenceid=8, compaction requested=false 2024-12-03T14:59:03,369 DEBUG [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1351): Waiting on 57c89a40077913e3c2dae3b7512725f9 2024-12-03T14:59:03,371 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/data/default/testRetryOutOfOrderScannerNextException/57c89a40077913e3c2dae3b7512725f9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T14:59:03,372 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. 2024-12-03T14:59:03,372 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 57c89a40077913e3c2dae3b7512725f9: Waiting for close lock at 1733237942922Running coprocessor pre-close hooks at 1733237942922Disabling compacts and flushes for region at 1733237942922Disabling writes for close at 1733237942922Obtaining lock to block concurrent updates at 1733237942922Preparing flush snapshotting stores in 57c89a40077913e3c2dae3b7512725f9 at 1733237942922Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733237942923 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. at 1733237942923Flushing 57c89a40077913e3c2dae3b7512725f9/testFamily: creating writer at 1733237942923Flushing 57c89a40077913e3c2dae3b7512725f9/testFamily: appending metadata at 1733237942936 (+13 ms)Flushing 57c89a40077913e3c2dae3b7512725f9/testFamily: closing flushed file at 1733237942936Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bf45ad4: reopening flushed file at 1733237943356 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 57c89a40077913e3c2dae3b7512725f9 in 444ms, sequenceid=8, compaction requested=false at 1733237943366 (+10 ms)Writing region close event to WAL at 1733237943367 (+1 ms)Running coprocessor post-close hooks at 1733237943372 (+5 ms)Closed at 1733237943372 2024-12-03T14:59:03,372 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733237897111.57c89a40077913e3c2dae3b7512725f9. 2024-12-03T14:59:03,569 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(976): stopping server a5d22df9eca2,45227,1733237844286; all regions closed. 2024-12-03T14:59:03,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741834_1010 (size=19479) 2024-12-03T14:59:03,581 DEBUG [RS:0;a5d22df9eca2:45227 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/oldWALs 2024-12-03T14:59:03,581 INFO [RS:0;a5d22df9eca2:45227 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a5d22df9eca2%2C45227%2C1733237844286.meta:.meta(num 1733237877341) 2024-12-03T14:59:03,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741833_1009 (size=20031) 2024-12-03T14:59:03,584 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/WALs/a5d22df9eca2,45227,1733237844286/a5d22df9eca2%2C45227%2C1733237844286.1733237876976 not finished, retry = 0 2024-12-03T14:59:03,691 DEBUG [RS:0;a5d22df9eca2:45227 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/oldWALs 2024-12-03T14:59:03,691 INFO [RS:0;a5d22df9eca2:45227 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a5d22df9eca2%2C45227%2C1733237844286:(num 1733237876976) 2024-12-03T14:59:03,691 DEBUG [RS:0;a5d22df9eca2:45227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:59:03,691 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T14:59:03,691 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T14:59:03,692 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.ChoreService(370): Chore service for: regionserver/a5d22df9eca2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T14:59:03,692 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T14:59:03,692 INFO [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T14:59:03,694 INFO [RS:0;a5d22df9eca2:45227 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45227 2024-12-03T14:59:03,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a5d22df9eca2,45227,1733237844286 2024-12-03T14:59:03,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T14:59:03,700 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T14:59:03,702 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a5d22df9eca2,45227,1733237844286] 2024-12-03T14:59:03,703 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a5d22df9eca2,45227,1733237844286 already deleted, retry=false 2024-12-03T14:59:03,703 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a5d22df9eca2,45227,1733237844286 expired; onlineServers=0 2024-12-03T14:59:03,703 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a5d22df9eca2,44239,1733237843695' ***** 2024-12-03T14:59:03,703 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T14:59:03,703 INFO [M:0;a5d22df9eca2:44239 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T14:59:03,703 INFO [M:0;a5d22df9eca2:44239 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T14:59:03,704 DEBUG [M:0;a5d22df9eca2:44239 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T14:59:03,704 DEBUG [M:0;a5d22df9eca2:44239 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T14:59:03,704 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T14:59:03,704 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237846796 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237846796,5,FailOnTimeoutGroup] 2024-12-03T14:59:03,704 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237846796 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237846796,5,FailOnTimeoutGroup] 2024-12-03T14:59:03,704 INFO [M:0;a5d22df9eca2:44239 {}] hbase.ChoreService(370): Chore service for: master/a5d22df9eca2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T14:59:03,704 INFO [M:0;a5d22df9eca2:44239 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T14:59:03,704 DEBUG [M:0;a5d22df9eca2:44239 {}] master.HMaster(1795): Stopping service threads 2024-12-03T14:59:03,704 INFO [M:0;a5d22df9eca2:44239 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T14:59:03,704 INFO [M:0;a5d22df9eca2:44239 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T14:59:03,705 INFO [M:0;a5d22df9eca2:44239 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T14:59:03,705 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T14:59:03,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T14:59:03,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:59:03,705 DEBUG [M:0;a5d22df9eca2:44239 {}] zookeeper.ZKUtil(347): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T14:59:03,705 WARN [M:0;a5d22df9eca2:44239 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T14:59:03,705 INFO [regionserver/a5d22df9eca2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T14:59:03,706 INFO [M:0;a5d22df9eca2:44239 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/.lastflushedseqids 2024-12-03T14:59:03,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741868_1044 (size=712) 2024-12-03T14:59:03,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:59:03,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45227-0x100a09600120001, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:59:03,804 INFO [RS:0;a5d22df9eca2:45227 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T14:59:03,804 INFO [RS:0;a5d22df9eca2:45227 {}] regionserver.HRegionServer(1031): Exiting; stopping=a5d22df9eca2,45227,1733237844286; zookeeper connection closed. 2024-12-03T14:59:03,805 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7df36da2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7df36da2 2024-12-03T14:59:03,806 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-03T14:59:04,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T14:59:04,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-12-03T14:59:04,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-03T14:59:04,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-03T14:59:04,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-03T14:59:04,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-03T14:59:04,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T14:59:04,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-03T14:59:04,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-03T14:59:04,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T14:59:04,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-03T14:59:04,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T14:59:04,117 INFO [M:0;a5d22df9eca2:44239 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T14:59:04,118 INFO [M:0;a5d22df9eca2:44239 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T14:59:04,118 DEBUG [M:0;a5d22df9eca2:44239 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T14:59:04,118 INFO [M:0;a5d22df9eca2:44239 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:59:04,118 DEBUG [M:0;a5d22df9eca2:44239 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:59:04,118 DEBUG [M:0;a5d22df9eca2:44239 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T14:59:04,118 DEBUG [M:0;a5d22df9eca2:44239 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:59:04,118 INFO [M:0;a5d22df9eca2:44239 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.07 KB heapSize=191.16 KB 2024-12-03T14:59:04,139 DEBUG [M:0;a5d22df9eca2:44239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fc684d5e18d141f0a635300b87b487d1 is 82, key is hbase:meta,,1/info:regioninfo/1733237877417/Put/seqid=0 2024-12-03T14:59:04,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741869_1045 (size=5672) 2024-12-03T14:59:04,545 INFO [M:0;a5d22df9eca2:44239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fc684d5e18d141f0a635300b87b487d1 2024-12-03T14:59:04,572 DEBUG [M:0;a5d22df9eca2:44239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/39b3cc54f85a42c08fb461a4a7b35915 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733237878980/Put/seqid=0 2024-12-03T14:59:04,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741870_1046 (size=14068) 2024-12-03T14:59:04,799 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:59:04,980 INFO [M:0;a5d22df9eca2:44239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.51 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/39b3cc54f85a42c08fb461a4a7b35915 2024-12-03T14:59:04,994 INFO [M:0;a5d22df9eca2:44239 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 39b3cc54f85a42c08fb461a4a7b35915 2024-12-03T14:59:05,011 DEBUG [M:0;a5d22df9eca2:44239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/36528bf0eb1b4f7b943248a2608754dd is 69, key is a5d22df9eca2,45227,1733237844286/rs:state/1733237846845/Put/seqid=0 2024-12-03T14:59:05,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741871_1047 (size=5156) 2024-12-03T14:59:05,416 INFO [M:0;a5d22df9eca2:44239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/36528bf0eb1b4f7b943248a2608754dd 2024-12-03T14:59:05,424 DEBUG [M:0;a5d22df9eca2:44239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fc684d5e18d141f0a635300b87b487d1 as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fc684d5e18d141f0a635300b87b487d1 2024-12-03T14:59:05,432 INFO [M:0;a5d22df9eca2:44239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fc684d5e18d141f0a635300b87b487d1, entries=8, sequenceid=319, filesize=5.5 K 2024-12-03T14:59:05,434 DEBUG [M:0;a5d22df9eca2:44239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/39b3cc54f85a42c08fb461a4a7b35915 as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/39b3cc54f85a42c08fb461a4a7b35915 2024-12-03T14:59:05,440 INFO [M:0;a5d22df9eca2:44239 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 39b3cc54f85a42c08fb461a4a7b35915 2024-12-03T14:59:05,440 INFO [M:0;a5d22df9eca2:44239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/39b3cc54f85a42c08fb461a4a7b35915, entries=33, sequenceid=319, filesize=13.7 K 2024-12-03T14:59:05,442 DEBUG [M:0;a5d22df9eca2:44239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/36528bf0eb1b4f7b943248a2608754dd as hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/36528bf0eb1b4f7b943248a2608754dd 2024-12-03T14:59:05,448 INFO [M:0;a5d22df9eca2:44239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39033/user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/36528bf0eb1b4f7b943248a2608754dd, entries=1, sequenceid=319, filesize=5.0 K 2024-12-03T14:59:05,449 INFO [M:0;a5d22df9eca2:44239 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.07 KB/166979, heapSize ~190.86 KB/195440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1331ms, sequenceid=319, compaction requested=false 2024-12-03T14:59:05,450 INFO [M:0;a5d22df9eca2:44239 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:59:05,451 DEBUG [M:0;a5d22df9eca2:44239 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733237944118Disabling compacts and flushes for region at 1733237944118Disabling writes for close at 1733237944118Obtaining lock to block concurrent updates at 1733237944119 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733237944119Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166979, getHeapSize=195680, getOffHeapSize=0, getCellsCount=366 at 1733237944119Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733237944120 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733237944120Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733237944138 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733237944138Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733237944554 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733237944572 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733237944572Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733237944994 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733237945011 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733237945011Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9cc78e8: reopening flushed file at 1733237945423 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cb83a7f: reopening flushed file at 1733237945433 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ab9b1b7: reopening flushed file at 1733237945440 (+7 ms)Finished flush of dataSize ~163.07 KB/166979, heapSize ~190.86 KB/195440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1331ms, sequenceid=319, compaction requested=false at 1733237945449 (+9 ms)Writing region close event to WAL at 1733237945450 (+1 ms)Closed at 1733237945450 2024-12-03T14:59:05,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44763 is added to blk_1073741830_1006 (size=188321) 2024-12-03T14:59:05,454 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/459d8329-80db-6df0-44d8-f5b4cdb09b0e/MasterData/WALs/a5d22df9eca2,44239,1733237843695/a5d22df9eca2%2C44239%2C1733237843695.1733237846310 not finished, retry = 0 2024-12-03T14:59:05,555 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T14:59:05,555 INFO [M:0;a5d22df9eca2:44239 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T14:59:05,555 INFO [M:0;a5d22df9eca2:44239 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44239 2024-12-03T14:59:05,555 INFO [M:0;a5d22df9eca2:44239 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T14:59:05,657 INFO [M:0;a5d22df9eca2:44239 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T14:59:05,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:59:05,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44239-0x100a09600120000, quorum=127.0.0.1:49750, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:59:05,663 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@258822a0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T14:59:05,666 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@514e86a2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T14:59:05,666 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T14:59:05,666 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c1ee62c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T14:59:05,666 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d304465{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/hadoop.log.dir/,STOPPED} 2024-12-03T14:59:05,669 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T14:59:05,669 WARN [BP-359429629-172.17.0.2-1733237841157 heartbeating to localhost/127.0.0.1:39033 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T14:59:05,669 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T14:59:05,669 WARN [BP-359429629-172.17.0.2-1733237841157 heartbeating to localhost/127.0.0.1:39033 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-359429629-172.17.0.2-1733237841157 (Datanode Uuid b10844da-bc93-4a91-a401-e2f18c7da2ef) service to localhost/127.0.0.1:39033 2024-12-03T14:59:05,671 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/cluster_e6ebabe5-9f0b-8d92-13b6-dc963c44e6c1/data/data1/current/BP-359429629-172.17.0.2-1733237841157 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T14:59:05,671 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/cluster_e6ebabe5-9f0b-8d92-13b6-dc963c44e6c1/data/data2/current/BP-359429629-172.17.0.2-1733237841157 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T14:59:05,672 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T14:59:05,679 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@59bae7b3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T14:59:05,679 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f237650{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T14:59:05,679 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T14:59:05,679 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45f63f41{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T14:59:05,680 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@660c56f7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e934bf58-6379-4861-5c6b-cddfbfc0c20a/hadoop.log.dir/,STOPPED} 2024-12-03T14:59:05,687 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T14:59:05,715 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down