2024-12-09 23:35:49,893 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-09 23:35:49,913 main DEBUG Took 0.017115 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 23:35:49,913 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 23:35:49,913 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 23:35:49,915 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 23:35:49,916 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:35:49,924 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 23:35:49,938 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:35:49,940 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:35:49,941 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:35:49,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:35:49,942 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:35:49,942 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:35:49,943 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:35:49,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:35:49,944 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:35:49,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:35:49,945 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:35:49,946 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:35:49,946 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:35:49,947 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:35:49,947 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:35:49,948 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:35:49,948 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:35:49,949 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:35:49,949 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:35:49,949 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:35:49,950 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:35:49,950 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:35:49,951 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:35:49,951 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 23:35:49,952 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:35:49,952 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 23:35:49,953 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 23:35:49,955 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 23:35:49,957 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 23:35:49,957 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 23:35:49,959 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 23:35:49,959 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 23:35:49,968 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 23:35:49,972 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 23:35:49,974 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 23:35:49,974 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 23:35:49,975 main DEBUG createAppenders(={Console}) 2024-12-09 23:35:49,976 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-09 23:35:49,976 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-09 23:35:49,977 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-09 23:35:49,978 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 23:35:49,978 main DEBUG OutputStream closed 2024-12-09 23:35:49,979 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 23:35:49,979 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 23:35:49,979 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-09 23:35:50,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 23:35:50,065 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 23:35:50,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 23:35:50,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 23:35:50,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 23:35:50,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 23:35:50,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 23:35:50,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 23:35:50,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 23:35:50,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 23:35:50,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 23:35:50,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 23:35:50,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 23:35:50,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 23:35:50,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 23:35:50,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 23:35:50,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 23:35:50,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 23:35:50,076 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 23:35:50,076 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-09 23:35:50,076 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 23:35:50,077 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-09T23:35:50,092 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-09 23:35:50,094 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 23:35:50,094 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T23:35:50,354 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3 2024-12-09T23:35:50,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T23:35:50,391 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/cluster_9d583092-22a2-df5c-e9ea-fff7e629db2e, deleteOnExit=true 2024-12-09T23:35:50,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T23:35:50,393 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/test.cache.data in system properties and HBase conf 2024-12-09T23:35:50,394 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T23:35:50,394 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/hadoop.log.dir in system properties and HBase conf 2024-12-09T23:35:50,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T23:35:50,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T23:35:50,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T23:35:50,492 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T23:35:50,600 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T23:35:50,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T23:35:50,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T23:35:50,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T23:35:50,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T23:35:50,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T23:35:50,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T23:35:50,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T23:35:50,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T23:35:50,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T23:35:50,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/nfs.dump.dir in system properties and HBase conf 2024-12-09T23:35:50,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/java.io.tmpdir in system properties and HBase conf 2024-12-09T23:35:50,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T23:35:50,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T23:35:50,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T23:35:51,707 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T23:35:51,783 INFO [Time-limited test {}] log.Log(170): Logging initialized @2624ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T23:35:51,861 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T23:35:51,925 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T23:35:51,947 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T23:35:51,948 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T23:35:51,949 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T23:35:51,963 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T23:35:51,966 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/hadoop.log.dir/,AVAILABLE} 2024-12-09T23:35:51,967 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T23:35:52,169 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/java.io.tmpdir/jetty-localhost-45633-hadoop-hdfs-3_4_1-tests_jar-_-any-16356743647288569927/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T23:35:52,180 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:45633} 2024-12-09T23:35:52,180 INFO [Time-limited test {}] server.Server(415): Started @3021ms 2024-12-09T23:35:52,751 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T23:35:52,758 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T23:35:52,759 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T23:35:52,760 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T23:35:52,760 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T23:35:52,761 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/hadoop.log.dir/,AVAILABLE} 2024-12-09T23:35:52,761 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T23:35:52,867 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/java.io.tmpdir/jetty-localhost-43365-hadoop-hdfs-3_4_1-tests_jar-_-any-13742034861438875289/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T23:35:52,868 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:43365} 2024-12-09T23:35:52,869 INFO [Time-limited test {}] server.Server(415): Started @3710ms 2024-12-09T23:35:52,924 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T23:35:54,012 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/cluster_9d583092-22a2-df5c-e9ea-fff7e629db2e/data/data2/current/BP-1725909415-172.17.0.3-1733787351209/current, will proceed with Du for space computation calculation, 2024-12-09T23:35:54,012 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/cluster_9d583092-22a2-df5c-e9ea-fff7e629db2e/data/data1/current/BP-1725909415-172.17.0.3-1733787351209/current, will proceed with Du for space computation calculation, 2024-12-09T23:35:54,042 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T23:35:54,090 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6f63e3c5fbe0e73b with lease ID 0xfe1f1987f0a5dc5d: Processing first storage report for DS-3dd38469-b8ea-4b86-9507-4ef3ec5fdfd0 from datanode DatanodeRegistration(127.0.0.1:33481, datanodeUuid=4dd69fc6-a6f0-4e54-9246-b2e1e64836c1, infoPort=32953, infoSecurePort=0, ipcPort=40421, storageInfo=lv=-57;cid=testClusterID;nsid=1983628430;c=1733787351209) 2024-12-09T23:35:54,091 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6f63e3c5fbe0e73b with lease ID 0xfe1f1987f0a5dc5d: from storage DS-3dd38469-b8ea-4b86-9507-4ef3ec5fdfd0 node DatanodeRegistration(127.0.0.1:33481, datanodeUuid=4dd69fc6-a6f0-4e54-9246-b2e1e64836c1, infoPort=32953, infoSecurePort=0, ipcPort=40421, storageInfo=lv=-57;cid=testClusterID;nsid=1983628430;c=1733787351209), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T23:35:54,091 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6f63e3c5fbe0e73b with lease ID 0xfe1f1987f0a5dc5d: Processing first storage report for DS-78fb79eb-36c3-43ba-8245-69a0529fe186 from datanode DatanodeRegistration(127.0.0.1:33481, datanodeUuid=4dd69fc6-a6f0-4e54-9246-b2e1e64836c1, infoPort=32953, infoSecurePort=0, ipcPort=40421, storageInfo=lv=-57;cid=testClusterID;nsid=1983628430;c=1733787351209) 2024-12-09T23:35:54,091 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6f63e3c5fbe0e73b with lease ID 0xfe1f1987f0a5dc5d: from storage DS-78fb79eb-36c3-43ba-8245-69a0529fe186 node DatanodeRegistration(127.0.0.1:33481, datanodeUuid=4dd69fc6-a6f0-4e54-9246-b2e1e64836c1, infoPort=32953, infoSecurePort=0, ipcPort=40421, storageInfo=lv=-57;cid=testClusterID;nsid=1983628430;c=1733787351209), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T23:35:54,114 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3 2024-12-09T23:35:54,189 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/cluster_9d583092-22a2-df5c-e9ea-fff7e629db2e/zookeeper_0, clientPort=52508, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/cluster_9d583092-22a2-df5c-e9ea-fff7e629db2e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/cluster_9d583092-22a2-df5c-e9ea-fff7e629db2e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T23:35:54,200 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52508 2024-12-09T23:35:54,213 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T23:35:54,216 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T23:35:54,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741825_1001 (size=7) 2024-12-09T23:35:54,854 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b with version=8 2024-12-09T23:35:54,854 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/hbase-staging 2024-12-09T23:35:54,940 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T23:35:55,222 INFO [Time-limited test {}] client.ConnectionUtils(128): master/aab77d931c0c:0 server-side Connection retries=9 2024-12-09T23:35:55,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T23:35:55,233 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T23:35:55,239 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T23:35:55,240 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T23:35:55,240 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T23:35:55,415 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T23:35:55,480 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T23:35:55,492 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T23:35:55,498 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T23:35:55,531 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 89015 (auto-detected) 2024-12-09T23:35:55,532 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-09T23:35:55,558 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33463 2024-12-09T23:35:55,581 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33463 connecting to ZooKeeper ensemble=127.0.0.1:52508 2024-12-09T23:35:55,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:334630x0, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T23:35:55,708 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33463-0x1000cef8b010000 connected 2024-12-09T23:35:55,811 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T23:35:55,814 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T23:35:55,824 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T23:35:55,828 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b, hbase.cluster.distributed=false 2024-12-09T23:35:55,854 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T23:35:55,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33463 2024-12-09T23:35:55,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33463 2024-12-09T23:35:55,860 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33463 2024-12-09T23:35:55,860 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33463 2024-12-09T23:35:55,861 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33463 2024-12-09T23:35:55,963 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/aab77d931c0c:0 server-side Connection retries=9 2024-12-09T23:35:55,964 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T23:35:55,965 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T23:35:55,965 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T23:35:55,965 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T23:35:55,965 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T23:35:55,968 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T23:35:55,970 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T23:35:55,970 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39071 2024-12-09T23:35:55,972 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39071 connecting to ZooKeeper ensemble=127.0.0.1:52508 2024-12-09T23:35:55,974 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T23:35:55,977 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T23:35:55,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:390710x0, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T23:35:55,989 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:390710x0, quorum=127.0.0.1:52508, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T23:35:55,989 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39071-0x1000cef8b010001 connected 2024-12-09T23:35:55,993 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T23:35:56,001 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T23:35:56,004 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T23:35:56,009 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T23:35:56,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39071 2024-12-09T23:35:56,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39071 2024-12-09T23:35:56,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39071 2024-12-09T23:35:56,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39071 2024-12-09T23:35:56,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39071 2024-12-09T23:35:56,028 DEBUG [M:0;aab77d931c0c:33463 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;aab77d931c0c:33463 2024-12-09T23:35:56,029 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/aab77d931c0c,33463,1733787355045 2024-12-09T23:35:56,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T23:35:56,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T23:35:56,043 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/aab77d931c0c,33463,1733787355045 2024-12-09T23:35:56,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:35:56,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T23:35:56,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:35:56,073 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T23:35:56,075 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/aab77d931c0c,33463,1733787355045 from backup master directory 2024-12-09T23:35:56,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/aab77d931c0c,33463,1733787355045 2024-12-09T23:35:56,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T23:35:56,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T23:35:56,089 WARN [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T23:35:56,090 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=aab77d931c0c,33463,1733787355045 2024-12-09T23:35:56,092 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T23:35:56,093 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T23:35:56,152 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/hbase.id] with ID: c00c5735-42ff-4fa9-8ada-d7bd3d6b104a 2024-12-09T23:35:56,152 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/.tmp/hbase.id 2024-12-09T23:35:56,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741826_1002 (size=42) 2024-12-09T23:35:56,565 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/.tmp/hbase.id]:[hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/hbase.id] 2024-12-09T23:35:56,617 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T23:35:56,622 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T23:35:56,641 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-12-09T23:35:56,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:35:56,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:35:56,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741827_1003 (size=196) 2024-12-09T23:35:57,126 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:35:57,128 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T23:35:57,140 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:35:57,143 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T23:35:57,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741828_1004 (size=1189) 2024-12-09T23:35:57,186 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store 2024-12-09T23:35:57,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741829_1005 (size=34) 2024-12-09T23:35:57,614 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T23:35:57,618 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:35:57,619 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T23:35:57,619 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:35:57,619 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:35:57,621 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T23:35:57,621 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:35:57,621 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:35:57,623 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733787357619Disabling compacts and flushes for region at 1733787357619Disabling writes for close at 1733787357621 (+2 ms)Writing region close event to WAL at 1733787357621Closed at 1733787357621 2024-12-09T23:35:57,625 WARN [master/aab77d931c0c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/.initializing 2024-12-09T23:35:57,625 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/WALs/aab77d931c0c,33463,1733787355045 2024-12-09T23:35:57,633 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T23:35:57,647 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aab77d931c0c%2C33463%2C1733787355045, suffix=, logDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/WALs/aab77d931c0c,33463,1733787355045, archiveDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/oldWALs, maxLogs=10 2024-12-09T23:35:57,667 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/WALs/aab77d931c0c,33463,1733787355045/aab77d931c0c%2C33463%2C1733787355045.1733787357651, exclude list is [], retry=0 2024-12-09T23:35:57,681 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33481,DS-3dd38469-b8ea-4b86-9507-4ef3ec5fdfd0,DISK] 2024-12-09T23:35:57,683 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T23:35:57,711 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/WALs/aab77d931c0c,33463,1733787355045/aab77d931c0c%2C33463%2C1733787355045.1733787357651 2024-12-09T23:35:57,712 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-09T23:35:57,713 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:35:57,713 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:35:57,715 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:35:57,716 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:35:57,748 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:35:57,770 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T23:35:57,773 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:35:57,775 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:35:57,776 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:35:57,779 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T23:35:57,780 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:35:57,781 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:35:57,781 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:35:57,783 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T23:35:57,784 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:35:57,785 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:35:57,785 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:35:57,788 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T23:35:57,788 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:35:57,789 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:35:57,789 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:35:57,793 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:35:57,794 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:35:57,800 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:35:57,801 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:35:57,804 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T23:35:57,808 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T23:35:57,813 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:35:57,814 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62781624, jitterRate=-0.06448090076446533}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T23:35:57,820 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733787357727Initializing all the Stores at 1733787357729 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733787357729Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787357730 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787357730Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787357730Cleaning up temporary data from old regions at 1733787357801 (+71 ms)Region opened successfully at 1733787357820 (+19 ms) 2024-12-09T23:35:57,821 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T23:35:57,853 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477b43eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aab77d931c0c/172.17.0.3:0 2024-12-09T23:35:57,880 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T23:35:57,890 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T23:35:57,890 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T23:35:57,892 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T23:35:57,894 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T23:35:57,899 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T23:35:57,899 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T23:35:57,928 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T23:35:57,938 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T23:35:57,994 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T23:35:57,997 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T23:35:57,999 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T23:35:58,008 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T23:35:58,012 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T23:35:58,018 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T23:35:58,029 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T23:35:58,030 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T23:35:58,040 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T23:35:58,061 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T23:35:58,071 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T23:35:58,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T23:35:58,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T23:35:58,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:35:58,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:35:58,087 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=aab77d931c0c,33463,1733787355045, sessionid=0x1000cef8b010000, setting cluster-up flag (Was=false) 2024-12-09T23:35:58,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:35:58,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:35:58,146 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T23:35:58,150 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=aab77d931c0c,33463,1733787355045 2024-12-09T23:35:58,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:35:58,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:35:58,209 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T23:35:58,213 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=aab77d931c0c,33463,1733787355045 2024-12-09T23:35:58,224 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T23:35:58,286 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T23:35:58,297 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T23:35:58,305 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T23:35:58,311 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: aab77d931c0c,33463,1733787355045 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T23:35:58,319 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(746): ClusterId : c00c5735-42ff-4fa9-8ada-d7bd3d6b104a 2024-12-09T23:35:58,320 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/aab77d931c0c:0, corePoolSize=5, maxPoolSize=5 2024-12-09T23:35:58,320 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/aab77d931c0c:0, corePoolSize=5, maxPoolSize=5 2024-12-09T23:35:58,320 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/aab77d931c0c:0, corePoolSize=5, maxPoolSize=5 2024-12-09T23:35:58,321 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/aab77d931c0c:0, corePoolSize=5, maxPoolSize=5 2024-12-09T23:35:58,321 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/aab77d931c0c:0, corePoolSize=10, maxPoolSize=10 2024-12-09T23:35:58,321 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/aab77d931c0c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:35:58,321 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/aab77d931c0c:0, corePoolSize=2, maxPoolSize=2 2024-12-09T23:35:58,321 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/aab77d931c0c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:35:58,322 DEBUG [RS:0;aab77d931c0c:39071 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T23:35:58,327 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733787388327 2024-12-09T23:35:58,329 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T23:35:58,330 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T23:35:58,330 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T23:35:58,331 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T23:35:58,335 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T23:35:58,335 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T23:35:58,336 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T23:35:58,336 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T23:35:58,336 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:35:58,336 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T23:35:58,337 DEBUG [RS:0;aab77d931c0c:39071 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T23:35:58,337 DEBUG [RS:0;aab77d931c0c:39071 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T23:35:58,337 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T23:35:58,341 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T23:35:58,342 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T23:35:58,343 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T23:35:58,345 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T23:35:58,345 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T23:35:58,347 DEBUG [RS:0;aab77d931c0c:39071 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T23:35:58,347 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/aab77d931c0c:0:becomeActiveMaster-HFileCleaner.large.0-1733787358346,5,FailOnTimeoutGroup] 2024-12-09T23:35:58,347 DEBUG [RS:0;aab77d931c0c:39071 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e013252, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aab77d931c0c/172.17.0.3:0 2024-12-09T23:35:58,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741831_1007 (size=1321) 2024-12-09T23:35:58,348 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/aab77d931c0c:0:becomeActiveMaster-HFileCleaner.small.0-1733787358347,5,FailOnTimeoutGroup] 2024-12-09T23:35:58,348 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T23:35:58,348 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T23:35:58,349 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T23:35:58,349 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T23:35:58,360 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;aab77d931c0c:39071 2024-12-09T23:35:58,363 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T23:35:58,363 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T23:35:58,363 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T23:35:58,365 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(2659): reportForDuty to master=aab77d931c0c,33463,1733787355045 with port=39071, startcode=1733787355928 2024-12-09T23:35:58,375 DEBUG [RS:0;aab77d931c0c:39071 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T23:35:58,405 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34535, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T23:35:58,412 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33463 {}] master.ServerManager(363): Checking decommissioned status of RegionServer aab77d931c0c,39071,1733787355928 2024-12-09T23:35:58,415 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33463 {}] master.ServerManager(517): Registering regionserver=aab77d931c0c,39071,1733787355928 2024-12-09T23:35:58,430 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b 2024-12-09T23:35:58,430 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37497 2024-12-09T23:35:58,430 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T23:35:58,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T23:35:58,441 DEBUG [RS:0;aab77d931c0c:39071 {}] zookeeper.ZKUtil(111): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/aab77d931c0c,39071,1733787355928 2024-12-09T23:35:58,441 WARN [RS:0;aab77d931c0c:39071 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T23:35:58,441 INFO [RS:0;aab77d931c0c:39071 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T23:35:58,442 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/WALs/aab77d931c0c,39071,1733787355928 2024-12-09T23:35:58,443 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [aab77d931c0c,39071,1733787355928] 2024-12-09T23:35:58,466 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T23:35:58,477 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T23:35:58,481 INFO [RS:0;aab77d931c0c:39071 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T23:35:58,481 INFO [RS:0;aab77d931c0c:39071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T23:35:58,482 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T23:35:58,488 INFO [RS:0;aab77d931c0c:39071 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T23:35:58,489 INFO [RS:0;aab77d931c0c:39071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T23:35:58,489 DEBUG [RS:0;aab77d931c0c:39071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/aab77d931c0c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:35:58,489 DEBUG [RS:0;aab77d931c0c:39071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/aab77d931c0c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:35:58,490 DEBUG [RS:0;aab77d931c0c:39071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:35:58,490 DEBUG [RS:0;aab77d931c0c:39071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/aab77d931c0c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:35:58,490 DEBUG [RS:0;aab77d931c0c:39071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/aab77d931c0c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:35:58,490 DEBUG [RS:0;aab77d931c0c:39071 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/aab77d931c0c:0, corePoolSize=2, maxPoolSize=2 2024-12-09T23:35:58,490 DEBUG [RS:0;aab77d931c0c:39071 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/aab77d931c0c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:35:58,490 DEBUG [RS:0;aab77d931c0c:39071 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/aab77d931c0c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:35:58,490 DEBUG [RS:0;aab77d931c0c:39071 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/aab77d931c0c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:35:58,491 DEBUG [RS:0;aab77d931c0c:39071 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/aab77d931c0c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:35:58,491 DEBUG [RS:0;aab77d931c0c:39071 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/aab77d931c0c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:35:58,491 DEBUG [RS:0;aab77d931c0c:39071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/aab77d931c0c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T23:35:58,491 DEBUG [RS:0;aab77d931c0c:39071 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/aab77d931c0c:0, corePoolSize=3, maxPoolSize=3 2024-12-09T23:35:58,491 DEBUG [RS:0;aab77d931c0c:39071 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/aab77d931c0c:0, corePoolSize=3, maxPoolSize=3 2024-12-09T23:35:58,492 INFO [RS:0;aab77d931c0c:39071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T23:35:58,492 INFO [RS:0;aab77d931c0c:39071 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T23:35:58,492 INFO [RS:0;aab77d931c0c:39071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T23:35:58,493 INFO [RS:0;aab77d931c0c:39071 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T23:35:58,493 INFO [RS:0;aab77d931c0c:39071 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T23:35:58,493 INFO [RS:0;aab77d931c0c:39071 {}] hbase.ChoreService(168): Chore ScheduledChore name=aab77d931c0c,39071,1733787355928-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T23:35:58,509 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T23:35:58,511 INFO [RS:0;aab77d931c0c:39071 {}] hbase.ChoreService(168): Chore ScheduledChore name=aab77d931c0c,39071,1733787355928-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T23:35:58,511 INFO [RS:0;aab77d931c0c:39071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T23:35:58,511 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.Replication(171): aab77d931c0c,39071,1733787355928 started 2024-12-09T23:35:58,527 INFO [RS:0;aab77d931c0c:39071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T23:35:58,528 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1482): Serving as aab77d931c0c,39071,1733787355928, RpcServer on aab77d931c0c/172.17.0.3:39071, sessionid=0x1000cef8b010001 2024-12-09T23:35:58,529 DEBUG [RS:0;aab77d931c0c:39071 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T23:35:58,529 DEBUG [RS:0;aab77d931c0c:39071 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager aab77d931c0c,39071,1733787355928 2024-12-09T23:35:58,529 DEBUG [RS:0;aab77d931c0c:39071 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aab77d931c0c,39071,1733787355928' 2024-12-09T23:35:58,529 DEBUG [RS:0;aab77d931c0c:39071 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T23:35:58,530 DEBUG [RS:0;aab77d931c0c:39071 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T23:35:58,531 DEBUG [RS:0;aab77d931c0c:39071 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T23:35:58,531 DEBUG [RS:0;aab77d931c0c:39071 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T23:35:58,531 DEBUG [RS:0;aab77d931c0c:39071 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager aab77d931c0c,39071,1733787355928 2024-12-09T23:35:58,531 DEBUG [RS:0;aab77d931c0c:39071 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aab77d931c0c,39071,1733787355928' 2024-12-09T23:35:58,531 DEBUG [RS:0;aab77d931c0c:39071 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T23:35:58,532 DEBUG [RS:0;aab77d931c0c:39071 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T23:35:58,533 DEBUG [RS:0;aab77d931c0c:39071 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T23:35:58,533 INFO [RS:0;aab77d931c0c:39071 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T23:35:58,533 INFO [RS:0;aab77d931c0c:39071 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T23:35:58,754 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T23:35:58,755 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b 2024-12-09T23:35:58,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741832_1008 (size=32) 2024-12-09T23:35:59,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:35:59,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T23:35:59,176 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T23:35:59,176 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:35:59,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:35:59,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T23:35:59,179 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T23:35:59,179 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:35:59,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:35:59,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T23:35:59,183 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T23:35:59,183 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:35:59,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:35:59,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T23:35:59,186 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T23:35:59,187 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:35:59,188 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:35:59,188 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T23:35:59,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740 2024-12-09T23:35:59,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740 2024-12-09T23:35:59,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T23:35:59,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T23:35:59,194 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T23:35:59,196 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T23:35:59,200 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:35:59,201 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70153568, jitterRate=0.045369625091552734}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T23:35:59,203 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733787359171Initializing all the Stores at 1733787359172 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733787359173 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733787359173Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787359173Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733787359173Cleaning up temporary data from old regions at 1733787359193 (+20 ms)Region opened successfully at 1733787359203 (+10 ms) 2024-12-09T23:35:59,203 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T23:35:59,203 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T23:35:59,203 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T23:35:59,203 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T23:35:59,203 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T23:35:59,205 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T23:35:59,205 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733787359203Disabling compacts and flushes for region at 1733787359203Disabling writes for close at 1733787359203Writing region close event to WAL at 1733787359204 (+1 ms)Closed at 1733787359204 2024-12-09T23:35:59,207 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T23:35:59,208 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T23:35:59,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T23:35:59,220 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T23:35:59,222 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T23:35:59,373 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:00,374 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:01,375 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:02,376 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:03,377 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:04,377 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:05,378 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:05,475 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T23:36:05,476 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T23:36:06,379 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:07,380 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:08,380 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:09,381 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:10,382 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:11,383 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:12,384 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:13,385 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:14,386 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:15,387 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:16,387 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:17,388 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:18,389 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:19,390 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:20,391 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:21,392 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:22,392 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:23,393 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:24,111 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T23:36:24,394 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:25,395 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:26,396 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:27,397 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:28,397 WARN [aab77d931c0c:33463 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T23:36:28,537 INFO [RS:0;aab77d931c0c:39071 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T23:36:28,541 INFO [RS:0;aab77d931c0c:39071 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aab77d931c0c%2C39071%2C1733787355928, suffix=, logDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/WALs/aab77d931c0c,39071,1733787355928, archiveDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/oldWALs, maxLogs=32 2024-12-09T23:36:28,556 DEBUG [RS:0;aab77d931c0c:39071 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/WALs/aab77d931c0c,39071,1733787355928/aab77d931c0c%2C39071%2C1733787355928.1733787388543, exclude list is [], retry=0 2024-12-09T23:36:28,560 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33481,DS-3dd38469-b8ea-4b86-9507-4ef3ec5fdfd0,DISK] 2024-12-09T23:36:28,563 INFO [RS:0;aab77d931c0c:39071 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/WALs/aab77d931c0c,39071,1733787355928/aab77d931c0c%2C39071%2C1733787355928.1733787388543 2024-12-09T23:36:28,564 DEBUG [RS:0;aab77d931c0c:39071 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-09T23:36:28,650 DEBUG [aab77d931c0c:33463 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-09T23:36:28,663 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:36:28,670 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as aab77d931c0c,39071,1733787355928, state=OPENING 2024-12-09T23:36:28,686 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T23:36:28,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:36:28,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:36:28,698 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T23:36:28,698 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T23:36:28,700 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T23:36:28,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=aab77d931c0c,39071,1733787355928}] 2024-12-09T23:36:28,882 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T23:36:28,886 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51165, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T23:36:28,895 INFO [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T23:36:28,896 INFO [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T23:36:28,896 INFO [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T23:36:28,899 INFO [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aab77d931c0c%2C39071%2C1733787355928.meta, suffix=.meta, logDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/WALs/aab77d931c0c,39071,1733787355928, archiveDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/oldWALs, maxLogs=32 2024-12-09T23:36:28,914 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/WALs/aab77d931c0c,39071,1733787355928/aab77d931c0c%2C39071%2C1733787355928.meta.1733787388900.meta, exclude list is [], retry=0 2024-12-09T23:36:28,918 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33481,DS-3dd38469-b8ea-4b86-9507-4ef3ec5fdfd0,DISK] 2024-12-09T23:36:28,921 INFO [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/WALs/aab77d931c0c,39071,1733787355928/aab77d931c0c%2C39071%2C1733787355928.meta.1733787388900.meta 2024-12-09T23:36:28,922 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-09T23:36:28,922 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:36:28,924 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T23:36:28,927 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T23:36:28,932 INFO [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T23:36:28,937 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T23:36:28,938 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:36:28,938 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T23:36:28,939 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T23:36:28,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T23:36:28,943 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T23:36:28,943 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:28,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:36:28,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T23:36:28,945 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T23:36:28,946 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:28,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:36:28,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T23:36:28,948 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T23:36:28,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:28,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:36:28,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T23:36:28,950 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T23:36:28,950 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:28,951 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T23:36:28,951 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T23:36:28,953 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740 2024-12-09T23:36:28,956 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740 2024-12-09T23:36:28,959 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T23:36:28,959 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T23:36:28,960 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T23:36:28,963 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T23:36:28,964 INFO [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58977127, jitterRate=-0.12117232382297516}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T23:36:28,965 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T23:36:28,966 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733787388939Writing region info on filesystem at 1733787388939Initializing all the Stores at 1733787388941 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733787388941Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733787388941Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787388941Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733787388941Cleaning up temporary data from old regions at 1733787388959 (+18 ms)Running coprocessor post-open hooks at 1733787388965 (+6 ms)Region opened successfully at 1733787388966 (+1 ms) 2024-12-09T23:36:28,972 INFO [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733787388873 2024-12-09T23:36:28,983 DEBUG [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T23:36:28,984 INFO [RS_OPEN_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T23:36:28,985 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:36:28,988 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as aab77d931c0c,39071,1733787355928, state=OPEN 2024-12-09T23:36:29,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T23:36:29,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T23:36:29,025 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T23:36:29,025 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T23:36:29,026 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=aab77d931c0c,39071,1733787355928 2024-12-09T23:36:29,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T23:36:29,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=aab77d931c0c,39071,1733787355928 in 323 msec 2024-12-09T23:36:29,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T23:36:29,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.8240 sec 2024-12-09T23:36:29,045 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T23:36:29,045 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T23:36:29,069 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:36:29,071 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aab77d931c0c,39071,1733787355928, seqNum=-1] 2024-12-09T23:36:29,095 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:36:29,098 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34585, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:36:29,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.8660 sec 2024-12-09T23:36:29,118 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733787389118, completionTime=-1 2024-12-09T23:36:29,121 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-09T23:36:29,121 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T23:36:29,143 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-09T23:36:29,143 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733787449143 2024-12-09T23:36:29,143 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733787509143 2024-12-09T23:36:29,143 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-12-09T23:36:29,145 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aab77d931c0c,33463,1733787355045-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T23:36:29,145 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aab77d931c0c,33463,1733787355045-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T23:36:29,145 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aab77d931c0c,33463,1733787355045-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T23:36:29,146 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-aab77d931c0c:33463, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T23:36:29,147 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T23:36:29,147 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T23:36:29,153 DEBUG [master/aab77d931c0c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T23:36:29,173 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.083sec 2024-12-09T23:36:29,174 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T23:36:29,176 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T23:36:29,177 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T23:36:29,177 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T23:36:29,178 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T23:36:29,178 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aab77d931c0c,33463,1733787355045-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T23:36:29,179 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aab77d931c0c,33463,1733787355045-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T23:36:29,185 DEBUG [master/aab77d931c0c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T23:36:29,186 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T23:36:29,186 INFO [master/aab77d931c0c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aab77d931c0c,33463,1733787355045-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T23:36:29,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77804624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:36:29,265 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T23:36:29,265 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T23:36:29,269 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aab77d931c0c,33463,-1 for getting cluster id 2024-12-09T23:36:29,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:36:29,279 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c00c5735-42ff-4fa9-8ada-d7bd3d6b104a' 2024-12-09T23:36:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:36:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c00c5735-42ff-4fa9-8ada-d7bd3d6b104a" 2024-12-09T23:36:29,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251b340d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:36:29,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aab77d931c0c,33463,-1] 2024-12-09T23:36:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:36:29,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:36:29,289 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:36:29,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fad6539, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:36:29,292 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:36:29,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aab77d931c0c,39071,1733787355928, seqNum=-1] 2024-12-09T23:36:29,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:36:29,302 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:36:29,320 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=aab77d931c0c,33463,1733787355045 2024-12-09T23:36:29,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68214a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:36:29,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aab77d931c0c,33463,-1 for getting cluster id 2024-12-09T23:36:29,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T23:36:29,324 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c00c5735-42ff-4fa9-8ada-d7bd3d6b104a' 2024-12-09T23:36:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T23:36:29,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c00c5735-42ff-4fa9-8ada-d7bd3d6b104a" 2024-12-09T23:36:29,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9487746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:36:29,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aab77d931c0c,33463,-1] 2024-12-09T23:36:29,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T23:36:29,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:36:29,329 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T23:36:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c4fe4b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T23:36:29,343 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=178, ProcessCount=11, AvailableMemoryMB=7650 2024-12-09T23:36:29,350 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T23:36:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is aab77d931c0c,33463,1733787355045 2024-12-09T23:36:29,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5e0f1a0 2024-12-09T23:36:29,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T23:36:29,361 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T23:36:29,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:36:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-09T23:36:29,397 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:36:29,399 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:29,399 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-09T23:36:29,404 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:36:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T23:36:29,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741835_1011 (size=428) 2024-12-09T23:36:29,837 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 71f1f612d5c5e6c68b902d1c7f2e431a, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b 2024-12-09T23:36:29,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741836_1012 (size=79) 2024-12-09T23:36:30,253 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:36:30,254 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 71f1f612d5c5e6c68b902d1c7f2e431a, disabling compactions & flushes 2024-12-09T23:36:30,254 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. 2024-12-09T23:36:30,254 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. 2024-12-09T23:36:30,255 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. after waiting 0 ms 2024-12-09T23:36:30,255 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. 2024-12-09T23:36:30,255 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. 2024-12-09T23:36:30,255 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 71f1f612d5c5e6c68b902d1c7f2e431a: Waiting for close lock at 1733787390254Disabling compacts and flushes for region at 1733787390254Disabling writes for close at 1733787390255 (+1 ms)Writing region close event to WAL at 1733787390255Closed at 1733787390255 2024-12-09T23:36:30,258 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:36:30,264 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733787390259"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733787390259"}]},"ts":"1733787390259"} 2024-12-09T23:36:30,270 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:36:30,273 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:36:30,275 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787390273"}]},"ts":"1733787390273"} 2024-12-09T23:36:30,280 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-09T23:36:30,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=71f1f612d5c5e6c68b902d1c7f2e431a, ASSIGN}] 2024-12-09T23:36:30,285 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=71f1f612d5c5e6c68b902d1c7f2e431a, ASSIGN 2024-12-09T23:36:30,287 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=71f1f612d5c5e6c68b902d1c7f2e431a, ASSIGN; state=OFFLINE, location=aab77d931c0c,39071,1733787355928; forceNewPlan=false, retain=false 2024-12-09T23:36:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T23:36:30,438 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=71f1f612d5c5e6c68b902d1c7f2e431a, regionState=OPENING, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:36:30,444 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=71f1f612d5c5e6c68b902d1c7f2e431a, ASSIGN because future has completed 2024-12-09T23:36:30,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 71f1f612d5c5e6c68b902d1c7f2e431a, server=aab77d931c0c,39071,1733787355928}] 2024-12-09T23:36:30,612 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. 2024-12-09T23:36:30,613 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 71f1f612d5c5e6c68b902d1c7f2e431a, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:36:30,613 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 71f1f612d5c5e6c68b902d1c7f2e431a 2024-12-09T23:36:30,613 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:36:30,614 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 71f1f612d5c5e6c68b902d1c7f2e431a 2024-12-09T23:36:30,614 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 71f1f612d5c5e6c68b902d1c7f2e431a 2024-12-09T23:36:30,616 INFO [StoreOpener-71f1f612d5c5e6c68b902d1c7f2e431a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 71f1f612d5c5e6c68b902d1c7f2e431a 2024-12-09T23:36:30,620 INFO [StoreOpener-71f1f612d5c5e6c68b902d1c7f2e431a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 71f1f612d5c5e6c68b902d1c7f2e431a columnFamilyName testFamily 2024-12-09T23:36:30,620 DEBUG [StoreOpener-71f1f612d5c5e6c68b902d1c7f2e431a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:30,621 INFO [StoreOpener-71f1f612d5c5e6c68b902d1c7f2e431a-1 {}] regionserver.HStore(327): Store=71f1f612d5c5e6c68b902d1c7f2e431a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:36:30,622 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 71f1f612d5c5e6c68b902d1c7f2e431a 2024-12-09T23:36:30,623 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/71f1f612d5c5e6c68b902d1c7f2e431a 2024-12-09T23:36:30,624 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/71f1f612d5c5e6c68b902d1c7f2e431a 2024-12-09T23:36:30,625 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 71f1f612d5c5e6c68b902d1c7f2e431a 2024-12-09T23:36:30,625 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 71f1f612d5c5e6c68b902d1c7f2e431a 2024-12-09T23:36:30,628 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 71f1f612d5c5e6c68b902d1c7f2e431a 2024-12-09T23:36:30,632 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/71f1f612d5c5e6c68b902d1c7f2e431a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:36:30,633 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 71f1f612d5c5e6c68b902d1c7f2e431a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75033911, jitterRate=0.11809240281581879}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T23:36:30,633 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 71f1f612d5c5e6c68b902d1c7f2e431a 2024-12-09T23:36:30,634 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 71f1f612d5c5e6c68b902d1c7f2e431a: Running coprocessor pre-open hook at 1733787390614Writing region info on filesystem at 1733787390614Initializing all the Stores at 1733787390616 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787390616Cleaning up temporary data from old regions at 1733787390625 (+9 ms)Running coprocessor post-open hooks at 1733787390633 (+8 ms)Region opened successfully at 1733787390634 (+1 ms) 2024-12-09T23:36:30,636 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a., pid=6, masterSystemTime=1733787390601 2024-12-09T23:36:30,640 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. 2024-12-09T23:36:30,640 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. 2024-12-09T23:36:30,641 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=71f1f612d5c5e6c68b902d1c7f2e431a, regionState=OPEN, openSeqNum=2, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:36:30,645 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 71f1f612d5c5e6c68b902d1c7f2e431a, server=aab77d931c0c,39071,1733787355928 because future has completed 2024-12-09T23:36:30,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T23:36:30,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 71f1f612d5c5e6c68b902d1c7f2e431a, server=aab77d931c0c,39071,1733787355928 in 203 msec 2024-12-09T23:36:30,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T23:36:30,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=71f1f612d5c5e6c68b902d1c7f2e431a, ASSIGN in 370 msec 2024-12-09T23:36:30,659 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:36:30,659 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787390659"}]},"ts":"1733787390659"} 2024-12-09T23:36:30,663 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-09T23:36:30,665 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:36:30,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2780 sec 2024-12-09T23:36:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T23:36:32,449 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-09T23:36:32,449 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-09T23:36:32,451 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:36:32,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-09T23:36:32,460 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:36:32,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-09T23:36:32,463 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T23:36:32,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aab77d931c0c,39071,1733787355928, seqNum=-1] 2024-12-09T23:36:32,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T23:36:32,469 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T23:36:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a., hostname=aab77d931c0c,39071,1733787355928, seqNum=2] 2024-12-09T23:36:32,489 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T23:36:32,492 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-09T23:36:32,497 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-09T23:36:32,501 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-09T23:36:32,503 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-09T23:36:32,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=7268760986272661512 2024-12-09T23:36:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39814 deadline: 1733787395502, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T23:36:32,528 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-09T23:36:32,530 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-09T23:36:32,531 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T23:36:32,533 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T23:36:32,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=7268760986272661514 2024-12-09T23:36:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39814 deadline: 1733787395532, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T23:36:32,538 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T23:36:32,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=7268760986272661515 2024-12-09T23:36:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39814 deadline: 1733787395537, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T23:36:32,542 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T23:36:32,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=7268760986272661516 2024-12-09T23:36:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39814 deadline: 1733787395542, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T23:36:32,546 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T23:36:32,556 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_182783648_22 at /127.0.0.1:42020 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=178 (was 178), ProcessCount=11 (was 11), AvailableMemoryMB=7560 (was 7650) 2024-12-09T23:36:32,564 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=178, ProcessCount=11, AvailableMemoryMB=7559 2024-12-09T23:36:32,567 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33463 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:36:32,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33463 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-09T23:36:32,570 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:36:32,571 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:32,571 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33463 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-09T23:36:32,572 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:36:32,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T23:36:32,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741837_1013 (size=414) 2024-12-09T23:36:32,987 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ce42a5d3c00cbd095e73d10231645ad4, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b 2024-12-09T23:36:32,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741838_1014 (size=65) 2024-12-09T23:36:33,400 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:36:33,401 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing ce42a5d3c00cbd095e73d10231645ad4, disabling compactions & flushes 2024-12-09T23:36:33,401 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. 2024-12-09T23:36:33,401 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. 2024-12-09T23:36:33,401 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. after waiting 0 ms 2024-12-09T23:36:33,401 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. 2024-12-09T23:36:33,401 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. 2024-12-09T23:36:33,401 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for ce42a5d3c00cbd095e73d10231645ad4: Waiting for close lock at 1733787393400Disabling compacts and flushes for region at 1733787393401 (+1 ms)Disabling writes for close at 1733787393401Writing region close event to WAL at 1733787393401Closed at 1733787393401 2024-12-09T23:36:33,404 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:36:33,404 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733787393404"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733787393404"}]},"ts":"1733787393404"} 2024-12-09T23:36:33,408 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:36:33,410 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:36:33,410 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787393410"}]},"ts":"1733787393410"} 2024-12-09T23:36:33,413 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-09T23:36:33,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ce42a5d3c00cbd095e73d10231645ad4, ASSIGN}] 2024-12-09T23:36:33,415 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ce42a5d3c00cbd095e73d10231645ad4, ASSIGN 2024-12-09T23:36:33,417 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ce42a5d3c00cbd095e73d10231645ad4, ASSIGN; state=OFFLINE, location=aab77d931c0c,39071,1733787355928; forceNewPlan=false, retain=false 2024-12-09T23:36:33,568 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ce42a5d3c00cbd095e73d10231645ad4, regionState=OPENING, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:36:33,571 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ce42a5d3c00cbd095e73d10231645ad4, ASSIGN because future has completed 2024-12-09T23:36:33,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce42a5d3c00cbd095e73d10231645ad4, server=aab77d931c0c,39071,1733787355928}] 2024-12-09T23:36:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T23:36:33,732 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. 2024-12-09T23:36:33,733 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ce42a5d3c00cbd095e73d10231645ad4, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:36:33,733 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:36:33,733 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:36:33,733 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:36:33,733 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:36:33,735 INFO [StoreOpener-ce42a5d3c00cbd095e73d10231645ad4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:36:33,737 INFO [StoreOpener-ce42a5d3c00cbd095e73d10231645ad4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce42a5d3c00cbd095e73d10231645ad4 columnFamilyName testFamily 2024-12-09T23:36:33,737 DEBUG [StoreOpener-ce42a5d3c00cbd095e73d10231645ad4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:33,738 INFO [StoreOpener-ce42a5d3c00cbd095e73d10231645ad4-1 {}] regionserver.HStore(327): Store=ce42a5d3c00cbd095e73d10231645ad4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:36:33,739 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:36:33,740 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScanner/ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:36:33,740 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScanner/ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:36:33,741 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:36:33,741 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:36:33,743 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:36:33,745 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScanner/ce42a5d3c00cbd095e73d10231645ad4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:36:33,746 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened ce42a5d3c00cbd095e73d10231645ad4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70272966, jitterRate=0.04714879393577576}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T23:36:33,746 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:36:33,747 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ce42a5d3c00cbd095e73d10231645ad4: Running coprocessor pre-open hook at 1733787393733Writing region info on filesystem at 1733787393733Initializing all the Stores at 1733787393735 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787393735Cleaning up temporary data from old regions at 1733787393741 (+6 ms)Running coprocessor post-open hooks at 1733787393746 (+5 ms)Region opened successfully at 1733787393747 (+1 ms) 2024-12-09T23:36:33,748 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4., pid=9, masterSystemTime=1733787393725 2024-12-09T23:36:33,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. 2024-12-09T23:36:33,752 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. 2024-12-09T23:36:33,752 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ce42a5d3c00cbd095e73d10231645ad4, regionState=OPEN, openSeqNum=2, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:36:33,755 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce42a5d3c00cbd095e73d10231645ad4, server=aab77d931c0c,39071,1733787355928 because future has completed 2024-12-09T23:36:33,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-09T23:36:33,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure ce42a5d3c00cbd095e73d10231645ad4, server=aab77d931c0c,39071,1733787355928 in 186 msec 2024-12-09T23:36:33,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T23:36:33,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ce42a5d3c00cbd095e73d10231645ad4, ASSIGN in 350 msec 2024-12-09T23:36:33,769 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:36:33,769 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787393769"}]},"ts":"1733787393769"} 2024-12-09T23:36:33,772 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-09T23:36:33,773 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:36:33,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2060 sec 2024-12-09T23:36:34,465 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:36:34,489 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-09T23:36:34,490 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T23:36:34,490 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-09T23:36:35,475 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T23:36:35,475 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T23:36:35,477 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T23:36:35,477 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T23:36:35,477 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-09T23:36:35,478 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T23:36:35,479 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-09T23:36:35,479 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-09T23:36:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T23:36:35,599 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-09T23:36:35,599 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-09T23:36:35,600 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:36:35,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-09T23:36:35,608 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:36:35,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-09T23:36:35,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4., hostname=aab77d931c0c,39071,1733787355928, seqNum=2] 2024-12-09T23:36:35,623 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T23:36:35,623 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-09T23:36:35,627 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-09T23:36:35,628 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-09T23:36:41,641 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4., hostname=aab77d931c0c,39071,1733787355928, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=aab77d931c0c:39071 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6011ms, rpcTimeout=5999ms 2024-12-09T23:36:41,642 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6011ms, rpcTimeout=5999ms 2024-12-09T23:36:41,642 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 from cache 2024-12-09T23:36:41,650 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=151 (was 178), ProcessCount=11 (was 11), AvailableMemoryMB=7404 (was 7559) 2024-12-09T23:36:41,658 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=151, ProcessCount=11, AvailableMemoryMB=7404 2024-12-09T23:36:41,660 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33463 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:36:41,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33463 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-09T23:36:41,664 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:36:41,664 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:41,664 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33463 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-09T23:36:41,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-09T23:36:41,666 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:36:41,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741839_1015 (size=412) 2024-12-09T23:36:42,082 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a7a609a0f8fdf69b8dc548a311c82eb2, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b 2024-12-09T23:36:42,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741840_1016 (size=63) 2024-12-09T23:36:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-12-09T23:36:42,500 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:36:42,501 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing a7a609a0f8fdf69b8dc548a311c82eb2, disabling compactions & flushes 2024-12-09T23:36:42,501 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. 2024-12-09T23:36:42,501 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. 2024-12-09T23:36:42,501 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. after waiting 0 ms 2024-12-09T23:36:42,502 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. 2024-12-09T23:36:42,502 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. 2024-12-09T23:36:42,502 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for a7a609a0f8fdf69b8dc548a311c82eb2: Waiting for close lock at 1733787402501Disabling compacts and flushes for region at 1733787402501Disabling writes for close at 1733787402502 (+1 ms)Writing region close event to WAL at 1733787402502Closed at 1733787402502 2024-12-09T23:36:42,508 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:36:42,508 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733787402508"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733787402508"}]},"ts":"1733787402508"} 2024-12-09T23:36:42,512 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:36:42,514 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:36:42,515 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787402514"}]},"ts":"1733787402514"} 2024-12-09T23:36:42,518 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-09T23:36:42,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=a7a609a0f8fdf69b8dc548a311c82eb2, ASSIGN}] 2024-12-09T23:36:42,521 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=a7a609a0f8fdf69b8dc548a311c82eb2, ASSIGN 2024-12-09T23:36:42,523 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=a7a609a0f8fdf69b8dc548a311c82eb2, ASSIGN; state=OFFLINE, location=aab77d931c0c,39071,1733787355928; forceNewPlan=false, retain=false 2024-12-09T23:36:42,674 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=a7a609a0f8fdf69b8dc548a311c82eb2, regionState=OPENING, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:36:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-09T23:36:42,681 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=a7a609a0f8fdf69b8dc548a311c82eb2, ASSIGN because future has completed 2024-12-09T23:36:42,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a609a0f8fdf69b8dc548a311c82eb2, server=aab77d931c0c,39071,1733787355928}] 2024-12-09T23:36:42,850 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. 2024-12-09T23:36:42,850 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => a7a609a0f8fdf69b8dc548a311c82eb2, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:36:42,851 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:36:42,851 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:36:42,851 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:36:42,851 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:36:42,853 INFO [StoreOpener-a7a609a0f8fdf69b8dc548a311c82eb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:36:42,855 INFO [StoreOpener-a7a609a0f8fdf69b8dc548a311c82eb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7a609a0f8fdf69b8dc548a311c82eb2 columnFamilyName testFamily 2024-12-09T23:36:42,855 DEBUG [StoreOpener-a7a609a0f8fdf69b8dc548a311c82eb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:42,856 INFO [StoreOpener-a7a609a0f8fdf69b8dc548a311c82eb2-1 {}] regionserver.HStore(327): Store=a7a609a0f8fdf69b8dc548a311c82eb2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:36:42,856 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:36:42,857 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNextAsync/a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:36:42,857 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNextAsync/a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:36:42,858 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:36:42,858 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:36:42,860 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:36:42,863 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNextAsync/a7a609a0f8fdf69b8dc548a311c82eb2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:36:42,864 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened a7a609a0f8fdf69b8dc548a311c82eb2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61654010, jitterRate=-0.08128365874290466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T23:36:42,864 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:36:42,865 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for a7a609a0f8fdf69b8dc548a311c82eb2: Running coprocessor pre-open hook at 1733787402851Writing region info on filesystem at 1733787402851Initializing all the Stores at 1733787402852 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787402853 (+1 ms)Cleaning up temporary data from old regions at 1733787402858 (+5 ms)Running coprocessor post-open hooks at 1733787402864 (+6 ms)Region opened successfully at 1733787402865 (+1 ms) 2024-12-09T23:36:42,866 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2., pid=12, masterSystemTime=1733787402837 2024-12-09T23:36:42,869 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. 2024-12-09T23:36:42,869 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. 2024-12-09T23:36:42,870 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=a7a609a0f8fdf69b8dc548a311c82eb2, regionState=OPEN, openSeqNum=2, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:36:42,873 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a609a0f8fdf69b8dc548a311c82eb2, server=aab77d931c0c,39071,1733787355928 because future has completed 2024-12-09T23:36:42,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-09T23:36:42,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure a7a609a0f8fdf69b8dc548a311c82eb2, server=aab77d931c0c,39071,1733787355928 in 193 msec 2024-12-09T23:36:42,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-09T23:36:42,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=a7a609a0f8fdf69b8dc548a311c82eb2, ASSIGN in 360 msec 2024-12-09T23:36:42,882 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:36:42,882 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787402882"}]},"ts":"1733787402882"} 2024-12-09T23:36:42,885 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-09T23:36:42,886 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:36:42,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2260 sec 2024-12-09T23:36:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-09T23:36:44,710 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-09T23:36:44,710 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-09T23:36:44,711 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:36:44,719 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-09T23:36:44,720 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:36:44,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-09T23:36:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2., hostname=aab77d931c0c,39071,1733787355928, seqNum=2] 2024-12-09T23:36:44,736 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T23:36:44,736 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-09T23:36:44,740 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-09T23:36:44,742 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-09T23:36:44,744 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-09T23:36:44,745 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-09T23:36:45,475 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-09T23:36:45,475 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T23:36:46,493 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-09T23:36:48,381 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:36:50,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39071 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '7268760986272661525' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T23:36:50,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39071 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:39814 deadline: 1733787416748, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '7268760986272661525' does not exist 2024-12-09T23:36:50,760 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to aab77d931c0c,39071,1733787355928 for closing scanner id = 7268760986272661525 for a7a609a0f8fdf69b8dc548a311c82eb2 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '7268760986272661525' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:36:50,763 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=142 (was 151), ProcessCount=11 (was 11), AvailableMemoryMB=7395 (was 7404) 2024-12-09T23:36:50,771 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=142, ProcessCount=11, AvailableMemoryMB=7395 2024-12-09T23:36:50,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:36:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-09T23:36:50,776 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:36:50,776 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:50,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-09T23:36:50,777 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:36:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T23:36:50,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741841_1017 (size=423) 2024-12-09T23:36:51,193 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 38a2db52d56427523d5f9315e91a5439, NAME => 'testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b 2024-12-09T23:36:51,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741842_1018 (size=74) 2024-12-09T23:36:51,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-12-09T23:36:51,610 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:36:51,611 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 38a2db52d56427523d5f9315e91a5439, disabling compactions & flushes 2024-12-09T23:36:51,611 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. 2024-12-09T23:36:51,611 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. 2024-12-09T23:36:51,612 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. after waiting 0 ms 2024-12-09T23:36:51,612 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. 2024-12-09T23:36:51,612 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. 2024-12-09T23:36:51,612 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 38a2db52d56427523d5f9315e91a5439: Waiting for close lock at 1733787411611Disabling compacts and flushes for region at 1733787411611Disabling writes for close at 1733787411612 (+1 ms)Writing region close event to WAL at 1733787411612Closed at 1733787411612 2024-12-09T23:36:51,654 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:36:51,654 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733787411654"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733787411654"}]},"ts":"1733787411654"} 2024-12-09T23:36:51,657 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:36:51,659 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:36:51,659 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787411659"}]},"ts":"1733787411659"} 2024-12-09T23:36:51,662 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-09T23:36:51,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=38a2db52d56427523d5f9315e91a5439, ASSIGN}] 2024-12-09T23:36:51,664 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=38a2db52d56427523d5f9315e91a5439, ASSIGN 2024-12-09T23:36:51,665 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=38a2db52d56427523d5f9315e91a5439, ASSIGN; state=OFFLINE, location=aab77d931c0c,39071,1733787355928; forceNewPlan=false, retain=false 2024-12-09T23:36:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T23:36:51,816 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=38a2db52d56427523d5f9315e91a5439, regionState=OPENING, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:36:51,822 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=38a2db52d56427523d5f9315e91a5439, ASSIGN because future has completed 2024-12-09T23:36:51,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38a2db52d56427523d5f9315e91a5439, server=aab77d931c0c,39071,1733787355928}] 2024-12-09T23:36:51,992 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. 2024-12-09T23:36:51,993 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 38a2db52d56427523d5f9315e91a5439, NAME => 'testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:36:51,993 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 38a2db52d56427523d5f9315e91a5439 2024-12-09T23:36:51,993 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:36:51,993 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 38a2db52d56427523d5f9315e91a5439 2024-12-09T23:36:51,993 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 38a2db52d56427523d5f9315e91a5439 2024-12-09T23:36:51,996 INFO [StoreOpener-38a2db52d56427523d5f9315e91a5439-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 38a2db52d56427523d5f9315e91a5439 2024-12-09T23:36:51,998 INFO [StoreOpener-38a2db52d56427523d5f9315e91a5439-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38a2db52d56427523d5f9315e91a5439 columnFamilyName testFamily 2024-12-09T23:36:51,999 DEBUG [StoreOpener-38a2db52d56427523d5f9315e91a5439-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:51,999 INFO [StoreOpener-38a2db52d56427523d5f9315e91a5439-1 {}] regionserver.HStore(327): Store=38a2db52d56427523d5f9315e91a5439/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:36:52,000 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 38a2db52d56427523d5f9315e91a5439 2024-12-09T23:36:52,001 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextException/38a2db52d56427523d5f9315e91a5439 2024-12-09T23:36:52,002 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextException/38a2db52d56427523d5f9315e91a5439 2024-12-09T23:36:52,002 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 38a2db52d56427523d5f9315e91a5439 2024-12-09T23:36:52,002 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 38a2db52d56427523d5f9315e91a5439 2024-12-09T23:36:52,006 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 38a2db52d56427523d5f9315e91a5439 2024-12-09T23:36:52,010 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextException/38a2db52d56427523d5f9315e91a5439/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:36:52,011 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 38a2db52d56427523d5f9315e91a5439; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71937625, jitterRate=0.071954146027565}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T23:36:52,011 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38a2db52d56427523d5f9315e91a5439 2024-12-09T23:36:52,012 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 38a2db52d56427523d5f9315e91a5439: Running coprocessor pre-open hook at 1733787411993Writing region info on filesystem at 1733787411993Initializing all the Stores at 1733787411995 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787411995Cleaning up temporary data from old regions at 1733787412002 (+7 ms)Running coprocessor post-open hooks at 1733787412011 (+9 ms)Region opened successfully at 1733787412012 (+1 ms) 2024-12-09T23:36:52,014 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439., pid=15, masterSystemTime=1733787411979 2024-12-09T23:36:52,017 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. 2024-12-09T23:36:52,018 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. 2024-12-09T23:36:52,019 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=38a2db52d56427523d5f9315e91a5439, regionState=OPEN, openSeqNum=2, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:36:52,022 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38a2db52d56427523d5f9315e91a5439, server=aab77d931c0c,39071,1733787355928 because future has completed 2024-12-09T23:36:52,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-09T23:36:52,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 38a2db52d56427523d5f9315e91a5439, server=aab77d931c0c,39071,1733787355928 in 201 msec 2024-12-09T23:36:52,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T23:36:52,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=38a2db52d56427523d5f9315e91a5439, ASSIGN in 365 msec 2024-12-09T23:36:52,032 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:36:52,032 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787412032"}]},"ts":"1733787412032"} 2024-12-09T23:36:52,035 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-09T23:36:52,036 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:36:52,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2630 sec 2024-12-09T23:36:52,497 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-09T23:36:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T23:36:53,809 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-09T23:36:53,809 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-09T23:36:53,810 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:36:53,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-09T23:36:53,818 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:36:53,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-09T23:36:53,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439., hostname=aab77d931c0c,39071,1733787355928, seqNum=2] 2024-12-09T23:36:53,835 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T23:36:53,835 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-09T23:36:53,838 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-09T23:36:53,841 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-09T23:36:53,842 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-09T23:36:53,843 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=7268760986272661529 2024-12-09T23:36:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39814 deadline: 1733787416842, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T23:36:53,846 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-09T23:36:53,847 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-09T23:36:53,848 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T23:36:53,850 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T23:36:53,850 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=7268760986272661531 2024-12-09T23:36:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39814 deadline: 1733787416849, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T23:36:53,853 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T23:36:53,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=7268760986272661532 2024-12-09T23:36:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39814 deadline: 1733787416853, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T23:36:53,857 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T23:36:53,858 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=7268760986272661533 2024-12-09T23:36:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39814 deadline: 1733787416856, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T23:36:53,860 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T23:36:53,868 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=226 (was 225) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_182783648_22 at /127.0.0.1:38238 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=139 (was 142), ProcessCount=11 (was 11), AvailableMemoryMB=7388 (was 7395) 2024-12-09T23:36:53,876 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=226, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=139, ProcessCount=11, AvailableMemoryMB=7388 2024-12-09T23:36:53,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:36:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-09T23:36:53,882 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:36:53,882 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:53,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-09T23:36:53,883 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:36:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-09T23:36:53,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741843_1019 (size=418) 2024-12-09T23:36:54,112 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T23:36:54,299 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 710180ad3e423dc7415481b84b6d232e, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b 2024-12-09T23:36:54,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741844_1020 (size=69) 2024-12-09T23:36:54,714 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:36:54,714 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 710180ad3e423dc7415481b84b6d232e, disabling compactions & flushes 2024-12-09T23:36:54,715 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. 2024-12-09T23:36:54,715 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. 2024-12-09T23:36:54,715 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. after waiting 0 ms 2024-12-09T23:36:54,715 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. 2024-12-09T23:36:54,715 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. 2024-12-09T23:36:54,716 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 710180ad3e423dc7415481b84b6d232e: Waiting for close lock at 1733787414714Disabling compacts and flushes for region at 1733787414714Disabling writes for close at 1733787414715 (+1 ms)Writing region close event to WAL at 1733787414715Closed at 1733787414715 2024-12-09T23:36:54,721 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:36:54,721 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733787414721"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733787414721"}]},"ts":"1733787414721"} 2024-12-09T23:36:54,725 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:36:54,727 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:36:54,727 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787414727"}]},"ts":"1733787414727"} 2024-12-09T23:36:54,731 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-09T23:36:54,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=710180ad3e423dc7415481b84b6d232e, ASSIGN}] 2024-12-09T23:36:54,734 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=710180ad3e423dc7415481b84b6d232e, ASSIGN 2024-12-09T23:36:54,735 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=710180ad3e423dc7415481b84b6d232e, ASSIGN; state=OFFLINE, location=aab77d931c0c,39071,1733787355928; forceNewPlan=false, retain=false 2024-12-09T23:36:54,887 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=710180ad3e423dc7415481b84b6d232e, regionState=OPENING, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:36:54,894 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=710180ad3e423dc7415481b84b6d232e, ASSIGN because future has completed 2024-12-09T23:36:54,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 710180ad3e423dc7415481b84b6d232e, server=aab77d931c0c,39071,1733787355928}] 2024-12-09T23:36:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-09T23:36:55,057 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. 2024-12-09T23:36:55,057 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 710180ad3e423dc7415481b84b6d232e, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:36:55,057 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 710180ad3e423dc7415481b84b6d232e 2024-12-09T23:36:55,058 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:36:55,058 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 710180ad3e423dc7415481b84b6d232e 2024-12-09T23:36:55,058 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 710180ad3e423dc7415481b84b6d232e 2024-12-09T23:36:55,060 INFO [StoreOpener-710180ad3e423dc7415481b84b6d232e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 710180ad3e423dc7415481b84b6d232e 2024-12-09T23:36:55,062 INFO [StoreOpener-710180ad3e423dc7415481b84b6d232e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 710180ad3e423dc7415481b84b6d232e columnFamilyName testFamily 2024-12-09T23:36:55,062 DEBUG [StoreOpener-710180ad3e423dc7415481b84b6d232e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:55,062 INFO [StoreOpener-710180ad3e423dc7415481b84b6d232e-1 {}] regionserver.HStore(327): Store=710180ad3e423dc7415481b84b6d232e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:36:55,063 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 710180ad3e423dc7415481b84b6d232e 2024-12-09T23:36:55,064 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScanner/710180ad3e423dc7415481b84b6d232e 2024-12-09T23:36:55,064 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScanner/710180ad3e423dc7415481b84b6d232e 2024-12-09T23:36:55,065 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 710180ad3e423dc7415481b84b6d232e 2024-12-09T23:36:55,065 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 710180ad3e423dc7415481b84b6d232e 2024-12-09T23:36:55,067 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 710180ad3e423dc7415481b84b6d232e 2024-12-09T23:36:55,070 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScanner/710180ad3e423dc7415481b84b6d232e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:36:55,071 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 710180ad3e423dc7415481b84b6d232e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62182997, jitterRate=-0.07340113818645477}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T23:36:55,071 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 710180ad3e423dc7415481b84b6d232e 2024-12-09T23:36:55,071 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 710180ad3e423dc7415481b84b6d232e: Running coprocessor pre-open hook at 1733787415058Writing region info on filesystem at 1733787415058Initializing all the Stores at 1733787415059 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787415059Cleaning up temporary data from old regions at 1733787415065 (+6 ms)Running coprocessor post-open hooks at 1733787415071 (+6 ms)Region opened successfully at 1733787415071 2024-12-09T23:36:55,072 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e., pid=18, masterSystemTime=1733787415050 2024-12-09T23:36:55,075 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. 2024-12-09T23:36:55,075 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. 2024-12-09T23:36:55,076 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=710180ad3e423dc7415481b84b6d232e, regionState=OPEN, openSeqNum=2, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:36:55,078 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 710180ad3e423dc7415481b84b6d232e, server=aab77d931c0c,39071,1733787355928 because future has completed 2024-12-09T23:36:55,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-09T23:36:55,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 710180ad3e423dc7415481b84b6d232e, server=aab77d931c0c,39071,1733787355928 in 185 msec 2024-12-09T23:36:55,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-09T23:36:55,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=710180ad3e423dc7415481b84b6d232e, ASSIGN in 352 msec 2024-12-09T23:36:55,087 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:36:55,087 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787415087"}]},"ts":"1733787415087"} 2024-12-09T23:36:55,089 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-09T23:36:55,091 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:36:55,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2130 sec 2024-12-09T23:36:55,475 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-09T23:36:55,475 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-09T23:36:55,478 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-09T23:36:55,478 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-09T23:36:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-09T23:36:56,928 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-09T23:36:56,928 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-09T23:36:56,928 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:36:56,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-09T23:36:56,938 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:36:56,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-09T23:36:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e., hostname=aab77d931c0c,39071,1733787355928, seqNum=2] 2024-12-09T23:36:56,951 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T23:36:56,951 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-09T23:36:56,954 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-09T23:36:56,956 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-09T23:36:57,498 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:36:57,966 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e., hostname=aab77d931c0c,39071,1733787355928, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=aab77d931c0c:39071 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-12-09T23:36:57,966 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-12-09T23:36:57,966 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 from cache 2024-12-09T23:36:58,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-12-09T23:36:58,501 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-09T23:36:58,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e., hostname=aab77d931c0c,39071,1733787355928, seqNum=2] 2024-12-09T23:36:58,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-09T23:36:59,956 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e., hostname=aab77d931c0c,39071,1733787355928, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=aab77d931c0c:39071 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=959ms, rpcTimeout=959ms 2024-12-09T23:36:59,956 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=959ms, rpcTimeout=959ms 2024-12-09T23:36:59,956 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 from cache 2024-12-09T23:36:59,972 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 226), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=128 (was 139), ProcessCount=11 (was 11), AvailableMemoryMB=7379 (was 7388) 2024-12-09T23:36:59,981 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=128, ProcessCount=11, AvailableMemoryMB=7378 2024-12-09T23:36:59,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:36:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-09T23:36:59,988 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:36:59,988 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:36:59,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-09T23:36:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T23:36:59,990 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:36:59,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741845_1021 (size=416) 2024-12-09T23:37:00,408 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9fe7e297263a8471832fe5e82d32a71f, NAME => 'testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b 2024-12-09T23:37:00,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741846_1022 (size=67) 2024-12-09T23:37:00,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-12-09T23:37:00,823 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:37:00,823 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 9fe7e297263a8471832fe5e82d32a71f, disabling compactions & flushes 2024-12-09T23:37:00,823 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. 2024-12-09T23:37:00,824 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. 2024-12-09T23:37:00,824 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. after waiting 0 ms 2024-12-09T23:37:00,824 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. 2024-12-09T23:37:00,824 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. 2024-12-09T23:37:00,825 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9fe7e297263a8471832fe5e82d32a71f: Waiting for close lock at 1733787420823Disabling compacts and flushes for region at 1733787420823Disabling writes for close at 1733787420824 (+1 ms)Writing region close event to WAL at 1733787420824Closed at 1733787420824 2024-12-09T23:37:00,828 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:37:00,828 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733787420828"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733787420828"}]},"ts":"1733787420828"} 2024-12-09T23:37:00,832 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:37:00,833 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:37:00,834 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787420833"}]},"ts":"1733787420833"} 2024-12-09T23:37:00,837 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-09T23:37:00,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=9fe7e297263a8471832fe5e82d32a71f, ASSIGN}] 2024-12-09T23:37:00,839 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=9fe7e297263a8471832fe5e82d32a71f, ASSIGN 2024-12-09T23:37:00,840 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=9fe7e297263a8471832fe5e82d32a71f, ASSIGN; state=OFFLINE, location=aab77d931c0c,39071,1733787355928; forceNewPlan=false, retain=false 2024-12-09T23:37:00,992 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=9fe7e297263a8471832fe5e82d32a71f, regionState=OPENING, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:37:01,000 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=9fe7e297263a8471832fe5e82d32a71f, ASSIGN because future has completed 2024-12-09T23:37:01,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fe7e297263a8471832fe5e82d32a71f, server=aab77d931c0c,39071,1733787355928}] 2024-12-09T23:37:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T23:37:01,164 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. 2024-12-09T23:37:01,164 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 9fe7e297263a8471832fe5e82d32a71f, NAME => 'testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:37:01,165 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 9fe7e297263a8471832fe5e82d32a71f 2024-12-09T23:37:01,165 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:37:01,165 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 9fe7e297263a8471832fe5e82d32a71f 2024-12-09T23:37:01,165 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 9fe7e297263a8471832fe5e82d32a71f 2024-12-09T23:37:01,167 INFO [StoreOpener-9fe7e297263a8471832fe5e82d32a71f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9fe7e297263a8471832fe5e82d32a71f 2024-12-09T23:37:01,169 INFO [StoreOpener-9fe7e297263a8471832fe5e82d32a71f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9fe7e297263a8471832fe5e82d32a71f columnFamilyName testFamily 2024-12-09T23:37:01,169 DEBUG [StoreOpener-9fe7e297263a8471832fe5e82d32a71f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:37:01,170 INFO [StoreOpener-9fe7e297263a8471832fe5e82d32a71f-1 {}] regionserver.HStore(327): Store=9fe7e297263a8471832fe5e82d32a71f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:37:01,170 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 9fe7e297263a8471832fe5e82d32a71f 2024-12-09T23:37:01,171 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNextAsync/9fe7e297263a8471832fe5e82d32a71f 2024-12-09T23:37:01,172 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNextAsync/9fe7e297263a8471832fe5e82d32a71f 2024-12-09T23:37:01,172 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 9fe7e297263a8471832fe5e82d32a71f 2024-12-09T23:37:01,172 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 9fe7e297263a8471832fe5e82d32a71f 2024-12-09T23:37:01,175 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 9fe7e297263a8471832fe5e82d32a71f 2024-12-09T23:37:01,178 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNextAsync/9fe7e297263a8471832fe5e82d32a71f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:37:01,179 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 9fe7e297263a8471832fe5e82d32a71f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66293638, jitterRate=-0.01214781403541565}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T23:37:01,179 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9fe7e297263a8471832fe5e82d32a71f 2024-12-09T23:37:01,180 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 9fe7e297263a8471832fe5e82d32a71f: Running coprocessor pre-open hook at 1733787421165Writing region info on filesystem at 1733787421165Initializing all the Stores at 1733787421167 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787421167Cleaning up temporary data from old regions at 1733787421173 (+6 ms)Running coprocessor post-open hooks at 1733787421179 (+6 ms)Region opened successfully at 1733787421180 (+1 ms) 2024-12-09T23:37:01,181 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f., pid=21, masterSystemTime=1733787421156 2024-12-09T23:37:01,183 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. 2024-12-09T23:37:01,183 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. 2024-12-09T23:37:01,184 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=9fe7e297263a8471832fe5e82d32a71f, regionState=OPEN, openSeqNum=2, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:37:01,186 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fe7e297263a8471832fe5e82d32a71f, server=aab77d931c0c,39071,1733787355928 because future has completed 2024-12-09T23:37:01,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-09T23:37:01,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 9fe7e297263a8471832fe5e82d32a71f, server=aab77d931c0c,39071,1733787355928 in 187 msec 2024-12-09T23:37:01,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-09T23:37:01,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=9fe7e297263a8471832fe5e82d32a71f, ASSIGN in 354 msec 2024-12-09T23:37:01,194 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:37:01,195 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787421194"}]},"ts":"1733787421194"} 2024-12-09T23:37:01,197 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-09T23:37:01,198 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:37:01,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2140 sec 2024-12-09T23:37:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T23:37:03,029 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-09T23:37:03,030 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-09T23:37:03,030 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:37:03,038 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-09T23:37:03,038 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:37:03,038 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-09T23:37:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f., hostname=aab77d931c0c,39071,1733787355928, seqNum=2] 2024-12-09T23:37:03,051 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T23:37:03,051 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-09T23:37:03,054 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-09T23:37:03,057 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-09T23:37:03,058 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-09T23:37:03,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-09T23:37:04,503 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-09T23:37:05,475 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-09T23:37:05,475 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T23:37:06,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39071 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '7268760986272661544' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T23:37:06,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39071 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:39814 deadline: 1733787427071, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '7268760986272661544' does not exist 2024-12-09T23:37:06,073 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to aab77d931c0c,39071,1733787355928 for closing scanner id = 7268760986272661544 for 9fe7e297263a8471832fe5e82d32a71f of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '7268760986272661544' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:37:06,079 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=117 (was 128), ProcessCount=11 (was 11), AvailableMemoryMB=7375 (was 7378) 2024-12-09T23:37:06,088 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=117, ProcessCount=11, AvailableMemoryMB=7375 2024-12-09T23:37:06,091 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:37:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-09T23:37:06,095 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:37:06,095 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:37:06,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-09T23:37:06,097 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:37:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-09T23:37:06,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741847_1023 (size=411) 2024-12-09T23:37:06,514 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 483fb2a88cb945cf193d9c5c908c57b9, NAME => 'testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b 2024-12-09T23:37:06,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741848_1024 (size=62) 2024-12-09T23:37:06,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-12-09T23:37:06,669 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:37:06,930 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:37:06,930 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 483fb2a88cb945cf193d9c5c908c57b9, disabling compactions & flushes 2024-12-09T23:37:06,930 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. 2024-12-09T23:37:06,931 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. 2024-12-09T23:37:06,931 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. after waiting 0 ms 2024-12-09T23:37:06,931 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. 2024-12-09T23:37:06,931 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. 2024-12-09T23:37:06,931 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 483fb2a88cb945cf193d9c5c908c57b9: Waiting for close lock at 1733787426930Disabling compacts and flushes for region at 1733787426930Disabling writes for close at 1733787426931 (+1 ms)Writing region close event to WAL at 1733787426931Closed at 1733787426931 2024-12-09T23:37:06,935 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:37:06,936 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733787426935"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733787426935"}]},"ts":"1733787426935"} 2024-12-09T23:37:06,941 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:37:06,943 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:37:06,943 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787426943"}]},"ts":"1733787426943"} 2024-12-09T23:37:06,946 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-09T23:37:06,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=483fb2a88cb945cf193d9c5c908c57b9, ASSIGN}] 2024-12-09T23:37:06,949 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=483fb2a88cb945cf193d9c5c908c57b9, ASSIGN 2024-12-09T23:37:06,950 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=483fb2a88cb945cf193d9c5c908c57b9, ASSIGN; state=OFFLINE, location=aab77d931c0c,39071,1733787355928; forceNewPlan=false, retain=false 2024-12-09T23:37:07,102 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=483fb2a88cb945cf193d9c5c908c57b9, regionState=OPENING, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:37:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-09T23:37:07,109 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=483fb2a88cb945cf193d9c5c908c57b9, ASSIGN because future has completed 2024-12-09T23:37:07,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 483fb2a88cb945cf193d9c5c908c57b9, server=aab77d931c0c,39071,1733787355928}] 2024-12-09T23:37:07,401 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. 2024-12-09T23:37:07,402 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 483fb2a88cb945cf193d9c5c908c57b9, NAME => 'testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:37:07,403 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 483fb2a88cb945cf193d9c5c908c57b9 2024-12-09T23:37:07,403 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:37:07,403 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 483fb2a88cb945cf193d9c5c908c57b9 2024-12-09T23:37:07,404 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 483fb2a88cb945cf193d9c5c908c57b9 2024-12-09T23:37:07,408 INFO [StoreOpener-483fb2a88cb945cf193d9c5c908c57b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 483fb2a88cb945cf193d9c5c908c57b9 2024-12-09T23:37:07,410 INFO [StoreOpener-483fb2a88cb945cf193d9c5c908c57b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 483fb2a88cb945cf193d9c5c908c57b9 columnFamilyName testFamily 2024-12-09T23:37:07,410 DEBUG [StoreOpener-483fb2a88cb945cf193d9c5c908c57b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:37:07,411 INFO [StoreOpener-483fb2a88cb945cf193d9c5c908c57b9-1 {}] regionserver.HStore(327): Store=483fb2a88cb945cf193d9c5c908c57b9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:37:07,411 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 483fb2a88cb945cf193d9c5c908c57b9 2024-12-09T23:37:07,413 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNext/483fb2a88cb945cf193d9c5c908c57b9 2024-12-09T23:37:07,413 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNext/483fb2a88cb945cf193d9c5c908c57b9 2024-12-09T23:37:07,414 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 483fb2a88cb945cf193d9c5c908c57b9 2024-12-09T23:37:07,414 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 483fb2a88cb945cf193d9c5c908c57b9 2024-12-09T23:37:07,416 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 483fb2a88cb945cf193d9c5c908c57b9 2024-12-09T23:37:07,419 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNext/483fb2a88cb945cf193d9c5c908c57b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:37:07,420 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 483fb2a88cb945cf193d9c5c908c57b9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70481297, jitterRate=0.05025316774845123}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T23:37:07,420 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 483fb2a88cb945cf193d9c5c908c57b9 2024-12-09T23:37:07,421 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 483fb2a88cb945cf193d9c5c908c57b9: Running coprocessor pre-open hook at 1733787427404Writing region info on filesystem at 1733787427404Initializing all the Stores at 1733787427407 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787427407Cleaning up temporary data from old regions at 1733787427414 (+7 ms)Running coprocessor post-open hooks at 1733787427421 (+7 ms)Region opened successfully at 1733787427421 2024-12-09T23:37:07,422 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9., pid=24, masterSystemTime=1733787427265 2024-12-09T23:37:07,425 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. 2024-12-09T23:37:07,425 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. 2024-12-09T23:37:07,425 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=483fb2a88cb945cf193d9c5c908c57b9, regionState=OPEN, openSeqNum=2, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:37:07,428 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 483fb2a88cb945cf193d9c5c908c57b9, server=aab77d931c0c,39071,1733787355928 because future has completed 2024-12-09T23:37:07,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-09T23:37:07,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 483fb2a88cb945cf193d9c5c908c57b9, server=aab77d931c0c,39071,1733787355928 in 320 msec 2024-12-09T23:37:07,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-09T23:37:07,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=483fb2a88cb945cf193d9c5c908c57b9, ASSIGN in 485 msec 2024-12-09T23:37:07,435 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:37:07,436 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787427435"}]},"ts":"1733787427435"} 2024-12-09T23:37:07,438 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-09T23:37:07,439 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:37:07,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.3480 sec 2024-12-09T23:37:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-09T23:37:09,130 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-09T23:37:09,130 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-09T23:37:09,131 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:37:09,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-09T23:37:09,139 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:37:09,139 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-09T23:37:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9., hostname=aab77d931c0c,39071,1733787355928, seqNum=2] 2024-12-09T23:37:09,153 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T23:37:09,154 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-09T23:37:09,158 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-09T23:37:09,161 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-09T23:37:09,163 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-09T23:37:09,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-09T23:37:09,248 INFO [master/aab77d931c0c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T23:37:09,248 INFO [master/aab77d931c0c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T23:37:10,505 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-09T23:37:12,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39071 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '7268760986272661548' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T23:37:12,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39071 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:39814 deadline: 1733787433168, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '7268760986272661548' does not exist 2024-12-09T23:37:12,175 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to aab77d931c0c,39071,1733787355928 for closing scanner id = 7268760986272661548 for 483fb2a88cb945cf193d9c5c908c57b9 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '7268760986272661548' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:37:12,182 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 117) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7375 (was 7375) 2024-12-09T23:37:12,190 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=7374 2024-12-09T23:37:12,193 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33463 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:37:12,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33463 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-09T23:37:12,197 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:37:12,197 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:37:12,197 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33463 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-09T23:37:12,198 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:37:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T23:37:12,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741849_1025 (size=419) 2024-12-09T23:37:12,613 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 00225674fa19c083605717af092c94b7, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b 2024-12-09T23:37:12,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-12-09T23:37:12,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741850_1026 (size=70) 2024-12-09T23:37:12,908 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:37:13,070 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:37:13,070 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 00225674fa19c083605717af092c94b7, disabling compactions & flushes 2024-12-09T23:37:13,070 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. 2024-12-09T23:37:13,070 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. 2024-12-09T23:37:13,070 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. after waiting 0 ms 2024-12-09T23:37:13,070 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. 2024-12-09T23:37:13,070 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. 2024-12-09T23:37:13,070 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 00225674fa19c083605717af092c94b7: Waiting for close lock at 1733787433070Disabling compacts and flushes for region at 1733787433070Disabling writes for close at 1733787433070Writing region close event to WAL at 1733787433070Closed at 1733787433070 2024-12-09T23:37:13,072 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:37:13,073 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733787433072"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733787433072"}]},"ts":"1733787433072"} 2024-12-09T23:37:13,078 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:37:13,080 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:37:13,080 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787433080"}]},"ts":"1733787433080"} 2024-12-09T23:37:13,083 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-09T23:37:13,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=00225674fa19c083605717af092c94b7, ASSIGN}] 2024-12-09T23:37:13,085 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=00225674fa19c083605717af092c94b7, ASSIGN 2024-12-09T23:37:13,086 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=00225674fa19c083605717af092c94b7, ASSIGN; state=OFFLINE, location=aab77d931c0c,39071,1733787355928; forceNewPlan=false, retain=false 2024-12-09T23:37:13,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T23:37:13,237 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=00225674fa19c083605717af092c94b7, regionState=OPENING, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:37:13,243 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=00225674fa19c083605717af092c94b7, ASSIGN because future has completed 2024-12-09T23:37:13,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00225674fa19c083605717af092c94b7, server=aab77d931c0c,39071,1733787355928}] 2024-12-09T23:37:13,410 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. 2024-12-09T23:37:13,410 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 00225674fa19c083605717af092c94b7, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:37:13,411 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 00225674fa19c083605717af092c94b7 2024-12-09T23:37:13,411 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:37:13,411 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 00225674fa19c083605717af092c94b7 2024-12-09T23:37:13,411 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 00225674fa19c083605717af092c94b7 2024-12-09T23:37:13,414 INFO [StoreOpener-00225674fa19c083605717af092c94b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 00225674fa19c083605717af092c94b7 2024-12-09T23:37:13,417 INFO [StoreOpener-00225674fa19c083605717af092c94b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00225674fa19c083605717af092c94b7 columnFamilyName testFamily 2024-12-09T23:37:13,417 DEBUG [StoreOpener-00225674fa19c083605717af092c94b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:37:13,418 INFO [StoreOpener-00225674fa19c083605717af092c94b7-1 {}] regionserver.HStore(327): Store=00225674fa19c083605717af092c94b7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:37:13,418 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 00225674fa19c083605717af092c94b7 2024-12-09T23:37:13,420 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/00225674fa19c083605717af092c94b7 2024-12-09T23:37:13,421 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/00225674fa19c083605717af092c94b7 2024-12-09T23:37:13,422 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 00225674fa19c083605717af092c94b7 2024-12-09T23:37:13,422 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 00225674fa19c083605717af092c94b7 2024-12-09T23:37:13,424 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 00225674fa19c083605717af092c94b7 2024-12-09T23:37:13,427 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/00225674fa19c083605717af092c94b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:37:13,428 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 00225674fa19c083605717af092c94b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65218127, jitterRate=-0.028174176812171936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T23:37:13,428 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00225674fa19c083605717af092c94b7 2024-12-09T23:37:13,429 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 00225674fa19c083605717af092c94b7: Running coprocessor pre-open hook at 1733787433412Writing region info on filesystem at 1733787433412Initializing all the Stores at 1733787433414 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787433414Cleaning up temporary data from old regions at 1733787433422 (+8 ms)Running coprocessor post-open hooks at 1733787433428 (+6 ms)Region opened successfully at 1733787433429 (+1 ms) 2024-12-09T23:37:13,430 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7., pid=27, masterSystemTime=1733787433398 2024-12-09T23:37:13,432 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. 2024-12-09T23:37:13,432 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. 2024-12-09T23:37:13,433 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=00225674fa19c083605717af092c94b7, regionState=OPEN, openSeqNum=2, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:37:13,436 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00225674fa19c083605717af092c94b7, server=aab77d931c0c,39071,1733787355928 because future has completed 2024-12-09T23:37:13,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-09T23:37:13,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 00225674fa19c083605717af092c94b7, server=aab77d931c0c,39071,1733787355928 in 193 msec 2024-12-09T23:37:13,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-09T23:37:13,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=00225674fa19c083605717af092c94b7, ASSIGN in 356 msec 2024-12-09T23:37:13,443 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:37:13,444 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787433443"}]},"ts":"1733787433443"} 2024-12-09T23:37:13,446 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-09T23:37:13,447 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:37:13,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2540 sec 2024-12-09T23:37:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T23:37:15,237 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-09T23:37:15,238 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-09T23:37:15,238 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:37:15,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-09T23:37:15,246 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:37:15,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-09T23:37:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7., hostname=aab77d931c0c,39071,1733787355928, seqNum=2] 2024-12-09T23:37:15,260 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T23:37:15,260 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-09T23:37:15,263 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-09T23:37:15,265 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-09T23:37:15,475 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-09T23:37:15,475 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-09T23:37:15,477 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-09T23:37:15,478 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T23:37:16,509 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-09T23:37:18,916 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:37:21,266 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7., hostname=aab77d931c0c,39071,1733787355928, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=aab77d931c0c:39071 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6002ms, rpcTimeout=5999ms 2024-12-09T23:37:21,266 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6002ms, rpcTimeout=5999ms 2024-12-09T23:37:21,267 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 from cache 2024-12-09T23:37:21,285 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=148 (was 140) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7372 (was 7374) 2024-12-09T23:37:21,294 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=148, ProcessCount=11, AvailableMemoryMB=7372 2024-12-09T23:37:21,297 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33463 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:37:21,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33463 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-09T23:37:21,300 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:37:21,300 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:37:21,300 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33463 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-09T23:37:21,302 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:37:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-09T23:37:21,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741851_1027 (size=407) 2024-12-09T23:37:21,716 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3b115f6a99189b4ae475ada1bbdcd290, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b 2024-12-09T23:37:21,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741852_1028 (size=58) 2024-12-09T23:37:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-12-09T23:37:22,128 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:37:22,129 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 3b115f6a99189b4ae475ada1bbdcd290, disabling compactions & flushes 2024-12-09T23:37:22,129 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. 2024-12-09T23:37:22,129 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. 2024-12-09T23:37:22,129 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. after waiting 0 ms 2024-12-09T23:37:22,129 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. 2024-12-09T23:37:22,129 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. 2024-12-09T23:37:22,130 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3b115f6a99189b4ae475ada1bbdcd290: Waiting for close lock at 1733787442128Disabling compacts and flushes for region at 1733787442128Disabling writes for close at 1733787442129 (+1 ms)Writing region close event to WAL at 1733787442129Closed at 1733787442129 2024-12-09T23:37:22,133 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:37:22,133 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733787442133"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733787442133"}]},"ts":"1733787442133"} 2024-12-09T23:37:22,137 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:37:22,139 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:37:22,139 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787442139"}]},"ts":"1733787442139"} 2024-12-09T23:37:22,142 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-09T23:37:22,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=3b115f6a99189b4ae475ada1bbdcd290, ASSIGN}] 2024-12-09T23:37:22,144 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=3b115f6a99189b4ae475ada1bbdcd290, ASSIGN 2024-12-09T23:37:22,145 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=3b115f6a99189b4ae475ada1bbdcd290, ASSIGN; state=OFFLINE, location=aab77d931c0c,39071,1733787355928; forceNewPlan=false, retain=false 2024-12-09T23:37:22,296 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=3b115f6a99189b4ae475ada1bbdcd290, regionState=OPENING, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:37:22,305 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=3b115f6a99189b4ae475ada1bbdcd290, ASSIGN because future has completed 2024-12-09T23:37:22,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3b115f6a99189b4ae475ada1bbdcd290, server=aab77d931c0c,39071,1733787355928}] 2024-12-09T23:37:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-09T23:37:22,472 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. 2024-12-09T23:37:22,472 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 3b115f6a99189b4ae475ada1bbdcd290, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:37:22,473 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 3b115f6a99189b4ae475ada1bbdcd290 2024-12-09T23:37:22,473 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:37:22,473 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 3b115f6a99189b4ae475ada1bbdcd290 2024-12-09T23:37:22,473 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 3b115f6a99189b4ae475ada1bbdcd290 2024-12-09T23:37:22,476 INFO [StoreOpener-3b115f6a99189b4ae475ada1bbdcd290-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3b115f6a99189b4ae475ada1bbdcd290 2024-12-09T23:37:22,478 INFO [StoreOpener-3b115f6a99189b4ae475ada1bbdcd290-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b115f6a99189b4ae475ada1bbdcd290 columnFamilyName testFamily 2024-12-09T23:37:22,478 DEBUG [StoreOpener-3b115f6a99189b4ae475ada1bbdcd290-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:37:22,479 INFO [StoreOpener-3b115f6a99189b4ae475ada1bbdcd290-1 {}] regionserver.HStore(327): Store=3b115f6a99189b4ae475ada1bbdcd290/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:37:22,480 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 3b115f6a99189b4ae475ada1bbdcd290 2024-12-09T23:37:22,481 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNext/3b115f6a99189b4ae475ada1bbdcd290 2024-12-09T23:37:22,482 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNext/3b115f6a99189b4ae475ada1bbdcd290 2024-12-09T23:37:22,482 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 3b115f6a99189b4ae475ada1bbdcd290 2024-12-09T23:37:22,482 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 3b115f6a99189b4ae475ada1bbdcd290 2024-12-09T23:37:22,486 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 3b115f6a99189b4ae475ada1bbdcd290 2024-12-09T23:37:22,490 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNext/3b115f6a99189b4ae475ada1bbdcd290/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:37:22,491 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 3b115f6a99189b4ae475ada1bbdcd290; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64737764, jitterRate=-0.03533214330673218}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T23:37:22,491 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3b115f6a99189b4ae475ada1bbdcd290 2024-12-09T23:37:22,492 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 3b115f6a99189b4ae475ada1bbdcd290: Running coprocessor pre-open hook at 1733787442473Writing region info on filesystem at 1733787442474 (+1 ms)Initializing all the Stores at 1733787442475 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787442475Cleaning up temporary data from old regions at 1733787442483 (+8 ms)Running coprocessor post-open hooks at 1733787442491 (+8 ms)Region opened successfully at 1733787442492 (+1 ms) 2024-12-09T23:37:22,494 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290., pid=30, masterSystemTime=1733787442459 2024-12-09T23:37:22,497 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. 2024-12-09T23:37:22,497 INFO [RS_OPEN_PRIORITY_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. 2024-12-09T23:37:22,498 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=3b115f6a99189b4ae475ada1bbdcd290, regionState=OPEN, openSeqNum=2, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:37:22,501 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3b115f6a99189b4ae475ada1bbdcd290, server=aab77d931c0c,39071,1733787355928 because future has completed 2024-12-09T23:37:22,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-09T23:37:22,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 3b115f6a99189b4ae475ada1bbdcd290, server=aab77d931c0c,39071,1733787355928 in 197 msec 2024-12-09T23:37:22,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-09T23:37:22,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=3b115f6a99189b4ae475ada1bbdcd290, ASSIGN in 363 msec 2024-12-09T23:37:22,510 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:37:22,510 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787442510"}]},"ts":"1733787442510"} 2024-12-09T23:37:22,511 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-09T23:37:22,513 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-09T23:37:22,514 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:37:22,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2170 sec 2024-12-09T23:37:24,112 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T23:37:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-09T23:37:24,329 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-09T23:37:24,330 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-09T23:37:24,330 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:37:24,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-09T23:37:24,341 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:37:24,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-09T23:37:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290., hostname=aab77d931c0c,39071,1733787355928, seqNum=2] 2024-12-09T23:37:24,353 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T23:37:24,353 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-09T23:37:24,356 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-09T23:37:24,358 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-09T23:37:24,359 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-09T23:37:24,359 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-09T23:37:25,475 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-09T23:37:25,475 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-09T23:37:27,977 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:37:30,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39071 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '7268760986272661556' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T23:37:30,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39071 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:39814 deadline: 1733787456367, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '7268760986272661556' does not exist 2024-12-09T23:37:30,374 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to aab77d931c0c,39071,1733787355928 for closing scanner id = 7268760986272661556 for 3b115f6a99189b4ae475ada1bbdcd290 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '7268760986272661556' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T23:37:30,381 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 148), ProcessCount=11 (was 11), AvailableMemoryMB=7376 (was 7372) - AvailableMemoryMB LEAK? - 2024-12-09T23:37:30,390 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=7376 2024-12-09T23:37:30,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T23:37:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-09T23:37:30,397 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T23:37:30,397 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:37:30,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-09T23:37:30,398 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T23:37:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-09T23:37:30,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741853_1029 (size=423) 2024-12-09T23:37:30,810 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 929bfbb576e7b4aa7d44b12e4d0d350f, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b 2024-12-09T23:37:30,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741854_1030 (size=74) 2024-12-09T23:37:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-12-09T23:37:30,973 DEBUG [master/aab77d931c0c:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-09T23:37:31,221 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:37:31,222 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 929bfbb576e7b4aa7d44b12e4d0d350f, disabling compactions & flushes 2024-12-09T23:37:31,222 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. 2024-12-09T23:37:31,222 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. 2024-12-09T23:37:31,222 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. after waiting 0 ms 2024-12-09T23:37:31,222 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. 2024-12-09T23:37:31,222 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. 2024-12-09T23:37:31,223 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 929bfbb576e7b4aa7d44b12e4d0d350f: Waiting for close lock at 1733787451221Disabling compacts and flushes for region at 1733787451222 (+1 ms)Disabling writes for close at 1733787451222Writing region close event to WAL at 1733787451222Closed at 1733787451222 2024-12-09T23:37:31,225 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T23:37:31,226 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733787451226"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733787451226"}]},"ts":"1733787451226"} 2024-12-09T23:37:31,230 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T23:37:31,232 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T23:37:31,232 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787451232"}]},"ts":"1733787451232"} 2024-12-09T23:37:31,235 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-09T23:37:31,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=929bfbb576e7b4aa7d44b12e4d0d350f, ASSIGN}] 2024-12-09T23:37:31,237 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=929bfbb576e7b4aa7d44b12e4d0d350f, ASSIGN 2024-12-09T23:37:31,238 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=929bfbb576e7b4aa7d44b12e4d0d350f, ASSIGN; state=OFFLINE, location=aab77d931c0c,39071,1733787355928; forceNewPlan=false, retain=false 2024-12-09T23:37:31,389 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=929bfbb576e7b4aa7d44b12e4d0d350f, regionState=OPENING, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:37:31,397 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=929bfbb576e7b4aa7d44b12e4d0d350f, ASSIGN because future has completed 2024-12-09T23:37:31,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 929bfbb576e7b4aa7d44b12e4d0d350f, server=aab77d931c0c,39071,1733787355928}] 2024-12-09T23:37:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-09T23:37:31,559 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. 2024-12-09T23:37:31,560 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 929bfbb576e7b4aa7d44b12e4d0d350f, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f.', STARTKEY => '', ENDKEY => ''} 2024-12-09T23:37:31,560 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:31,561 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T23:37:31,561 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:31,561 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:31,564 INFO [StoreOpener-929bfbb576e7b4aa7d44b12e4d0d350f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:31,567 INFO [StoreOpener-929bfbb576e7b4aa7d44b12e4d0d350f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 929bfbb576e7b4aa7d44b12e4d0d350f columnFamilyName testFamily 2024-12-09T23:37:31,567 DEBUG [StoreOpener-929bfbb576e7b4aa7d44b12e4d0d350f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T23:37:31,568 INFO [StoreOpener-929bfbb576e7b4aa7d44b12e4d0d350f-1 {}] regionserver.HStore(327): Store=929bfbb576e7b4aa7d44b12e4d0d350f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T23:37:31,568 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:31,569 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScannerAsync/929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:31,570 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScannerAsync/929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:31,571 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:31,571 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:31,574 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:31,577 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScannerAsync/929bfbb576e7b4aa7d44b12e4d0d350f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T23:37:31,577 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 929bfbb576e7b4aa7d44b12e4d0d350f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59030095, jitterRate=-0.12038303911685944}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T23:37:31,577 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:31,578 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 929bfbb576e7b4aa7d44b12e4d0d350f: Running coprocessor pre-open hook at 1733787451561Writing region info on filesystem at 1733787451561Initializing all the Stores at 1733787451564 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733787451564Cleaning up temporary data from old regions at 1733787451571 (+7 ms)Running coprocessor post-open hooks at 1733787451577 (+6 ms)Region opened successfully at 1733787451578 (+1 ms) 2024-12-09T23:37:31,579 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., pid=33, masterSystemTime=1733787451550 2024-12-09T23:37:31,582 DEBUG [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. 2024-12-09T23:37:31,582 INFO [RS_OPEN_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. 2024-12-09T23:37:31,582 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=929bfbb576e7b4aa7d44b12e4d0d350f, regionState=OPEN, openSeqNum=2, regionLocation=aab77d931c0c,39071,1733787355928 2024-12-09T23:37:31,585 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 929bfbb576e7b4aa7d44b12e4d0d350f, server=aab77d931c0c,39071,1733787355928 because future has completed 2024-12-09T23:37:31,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-09T23:37:31,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 929bfbb576e7b4aa7d44b12e4d0d350f, server=aab77d931c0c,39071,1733787355928 in 189 msec 2024-12-09T23:37:31,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-09T23:37:31,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=929bfbb576e7b4aa7d44b12e4d0d350f, ASSIGN in 353 msec 2024-12-09T23:37:31,591 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T23:37:31,591 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733787451591"}]},"ts":"1733787451591"} 2024-12-09T23:37:31,593 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-09T23:37:31,594 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T23:37:31,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2010 sec 2024-12-09T23:37:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-09T23:37:33,439 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-09T23:37:33,439 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-09T23:37:33,440 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:37:33,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-09T23:37:33,451 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T23:37:33,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-09T23:37:33,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., hostname=aab77d931c0c,39071,1733787355928, seqNum=2] 2024-12-09T23:37:33,463 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T23:37:33,463 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-09T23:37:33,467 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-09T23:37:33,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-09T23:37:34,476 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., hostname=aab77d931c0c,39071,1733787355928, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=aab77d931c0c:39071 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-12-09T23:37:34,476 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-12-09T23:37:34,477 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 from cache 2024-12-09T23:37:34,513 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-09T23:37:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-12-09T23:37:35,475 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-12-09T23:37:35,475 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T23:37:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., hostname=aab77d931c0c,39071,1733787355928, seqNum=2] 2024-12-09T23:37:35,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39071 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-09T23:37:36,476 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., hostname=aab77d931c0c,39071,1733787355928, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=aab77d931c0c:39071 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=982ms, rpcTimeout=974ms 2024-12-09T23:37:36,476 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=982ms, rpcTimeout=974ms 2024-12-09T23:37:36,476 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., hostname=aab77d931c0c,39071,1733787355928, seqNum=2 from cache 2024-12-09T23:37:36,489 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=115 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=7375 (was 7376) 2024-12-09T23:37:36,490 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T23:37:36,490 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T23:37:36,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:37:36,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:37:36,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:37:36,496 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T23:37:36,497 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T23:37:36,497 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T23:37:36,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:37:36,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:37:36,497 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T23:37:36,497 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T23:37:36,497 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1279614773, stopped=false 2024-12-09T23:37:36,498 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=aab77d931c0c,33463,1733787355045 2024-12-09T23:37:36,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T23:37:36,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T23:37:36,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:37:36,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:37:36,576 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T23:37:36,579 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T23:37:36,579 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T23:37:36,579 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T23:37:36,579 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T23:37:36,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:37:36,580 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'aab77d931c0c,39071,1733787355928' ***** 2024-12-09T23:37:36,580 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T23:37:36,581 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T23:37:36,581 INFO [RS:0;aab77d931c0c:39071 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T23:37:36,581 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T23:37:36,582 INFO [RS:0;aab77d931c0c:39071 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T23:37:36,582 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(3091): Received CLOSE for 483fb2a88cb945cf193d9c5c908c57b9 2024-12-09T23:37:36,583 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(3091): Received CLOSE for 3b115f6a99189b4ae475ada1bbdcd290 2024-12-09T23:37:36,583 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(3091): Received CLOSE for 71f1f612d5c5e6c68b902d1c7f2e431a 2024-12-09T23:37:36,583 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(3091): Received CLOSE for 38a2db52d56427523d5f9315e91a5439 2024-12-09T23:37:36,583 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(3091): Received CLOSE for 9fe7e297263a8471832fe5e82d32a71f 2024-12-09T23:37:36,583 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(3091): Received CLOSE for ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:37:36,583 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(3091): Received CLOSE for 00225674fa19c083605717af092c94b7 2024-12-09T23:37:36,583 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(3091): Received CLOSE for a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:37:36,583 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(3091): Received CLOSE for 929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:36,583 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 483fb2a88cb945cf193d9c5c908c57b9, disabling compactions & flushes 2024-12-09T23:37:36,583 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(3091): Received CLOSE for 710180ad3e423dc7415481b84b6d232e 2024-12-09T23:37:36,583 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. 2024-12-09T23:37:36,583 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(959): stopping server aab77d931c0c,39071,1733787355928 2024-12-09T23:37:36,583 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. 2024-12-09T23:37:36,583 INFO [RS:0;aab77d931c0c:39071 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T23:37:36,583 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. after waiting 0 ms 2024-12-09T23:37:36,583 INFO [RS:0;aab77d931c0c:39071 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;aab77d931c0c:39071. 2024-12-09T23:37:36,583 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. 2024-12-09T23:37:36,584 DEBUG [RS:0;aab77d931c0c:39071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T23:37:36,584 DEBUG [RS:0;aab77d931c0c:39071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:37:36,584 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T23:37:36,584 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T23:37:36,584 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T23:37:36,584 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T23:37:36,584 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-09T23:37:36,584 DEBUG [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T23:37:36,585 INFO [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T23:37:36,585 DEBUG [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T23:37:36,585 DEBUG [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T23:37:36,585 DEBUG [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T23:37:36,585 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 1588230740, 38a2db52d56427523d5f9315e91a5439, 3b115f6a99189b4ae475ada1bbdcd290, 483fb2a88cb945cf193d9c5c908c57b9, 710180ad3e423dc7415481b84b6d232e, 71f1f612d5c5e6c68b902d1c7f2e431a, 929bfbb576e7b4aa7d44b12e4d0d350f, 9fe7e297263a8471832fe5e82d32a71f, a7a609a0f8fdf69b8dc548a311c82eb2, ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:37:36,588 INFO [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-09T23:37:36,588 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 483fb2a88cb945cf193d9c5c908c57b9 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T23:37:36,644 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNext/483fb2a88cb945cf193d9c5c908c57b9/.tmp/testFamily/93467622a5b1482e8d5d43cfa6cf7b4d is 61, key is row-0/testFamily:testQualifier/1733787429143/Put/seqid=0 2024-12-09T23:37:36,648 DEBUG [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/.tmp/info/f2365a4d7e704414ac2da3b1fae26549 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a./info:regioninfo/1733787390641/Put/seqid=0 2024-12-09T23:37:36,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741855_1031 (size=20564) 2024-12-09T23:37:36,655 INFO [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/.tmp/info/f2365a4d7e704414ac2da3b1fae26549 2024-12-09T23:37:36,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741856_1032 (size=5049) 2024-12-09T23:37:36,657 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNext/483fb2a88cb945cf193d9c5c908c57b9/.tmp/testFamily/93467622a5b1482e8d5d43cfa6cf7b4d 2024-12-09T23:37:36,698 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNext/483fb2a88cb945cf193d9c5c908c57b9/.tmp/testFamily/93467622a5b1482e8d5d43cfa6cf7b4d as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNext/483fb2a88cb945cf193d9c5c908c57b9/testFamily/93467622a5b1482e8d5d43cfa6cf7b4d 2024-12-09T23:37:36,707 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNext/483fb2a88cb945cf193d9c5c908c57b9/testFamily/93467622a5b1482e8d5d43cfa6cf7b4d, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T23:37:36,715 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 483fb2a88cb945cf193d9c5c908c57b9 in 127ms, sequenceid=8, compaction requested=false 2024-12-09T23:37:36,716 DEBUG [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/.tmp/ns/c34d0638e82b4245a7c8d5ddcbfa5f43 is 43, key is default/ns:d/1733787389102/Put/seqid=0 2024-12-09T23:37:36,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741857_1033 (size=5153) 2024-12-09T23:37:36,722 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNext/483fb2a88cb945cf193d9c5c908c57b9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T23:37:36,722 INFO [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/.tmp/ns/c34d0638e82b4245a7c8d5ddcbfa5f43 2024-12-09T23:37:36,724 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. 2024-12-09T23:37:36,725 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 483fb2a88cb945cf193d9c5c908c57b9: Waiting for close lock at 1733787456583Running coprocessor pre-close hooks at 1733787456583Disabling compacts and flushes for region at 1733787456583Disabling writes for close at 1733787456583Obtaining lock to block concurrent updates at 1733787456588 (+5 ms)Preparing flush snapshotting stores in 483fb2a88cb945cf193d9c5c908c57b9 at 1733787456588Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733787456594 (+6 ms)Flushing stores of testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. at 1733787456595 (+1 ms)Flushing 483fb2a88cb945cf193d9c5c908c57b9/testFamily: creating writer at 1733787456597 (+2 ms)Flushing 483fb2a88cb945cf193d9c5c908c57b9/testFamily: appending metadata at 1733787456637 (+40 ms)Flushing 483fb2a88cb945cf193d9c5c908c57b9/testFamily: closing flushed file at 1733787456640 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c037f01: reopening flushed file at 1733787456696 (+56 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 483fb2a88cb945cf193d9c5c908c57b9 in 127ms, sequenceid=8, compaction requested=false at 1733787456715 (+19 ms)Writing region close event to WAL at 1733787456717 (+2 ms)Running coprocessor post-close hooks at 1733787456723 (+6 ms)Closed at 1733787456724 (+1 ms) 2024-12-09T23:37:36,725 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733787426091.483fb2a88cb945cf193d9c5c908c57b9. 2024-12-09T23:37:36,725 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3b115f6a99189b4ae475ada1bbdcd290, disabling compactions & flushes 2024-12-09T23:37:36,726 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. 2024-12-09T23:37:36,726 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. 2024-12-09T23:37:36,726 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. after waiting 0 ms 2024-12-09T23:37:36,726 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. 2024-12-09T23:37:36,726 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3b115f6a99189b4ae475ada1bbdcd290 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T23:37:36,744 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNext/3b115f6a99189b4ae475ada1bbdcd290/.tmp/testFamily/e37ef69477ab4b56a59808087b3d0ae4 is 61, key is row-0/testFamily:testQualifier/1733787444346/Put/seqid=0 2024-12-09T23:37:36,748 DEBUG [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/.tmp/table/431ab100aec1406191d5766372e0761f is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733787390659/Put/seqid=0 2024-12-09T23:37:36,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741858_1034 (size=5049) 2024-12-09T23:37:36,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741859_1035 (size=6699) 2024-12-09T23:37:36,785 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 1588230740, 38a2db52d56427523d5f9315e91a5439, 3b115f6a99189b4ae475ada1bbdcd290, 710180ad3e423dc7415481b84b6d232e, 71f1f612d5c5e6c68b902d1c7f2e431a, 929bfbb576e7b4aa7d44b12e4d0d350f, 9fe7e297263a8471832fe5e82d32a71f, a7a609a0f8fdf69b8dc548a311c82eb2, ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:37:36,985 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 1588230740, 38a2db52d56427523d5f9315e91a5439, 3b115f6a99189b4ae475ada1bbdcd290, 710180ad3e423dc7415481b84b6d232e, 71f1f612d5c5e6c68b902d1c7f2e431a, 929bfbb576e7b4aa7d44b12e4d0d350f, 9fe7e297263a8471832fe5e82d32a71f, a7a609a0f8fdf69b8dc548a311c82eb2, ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:37:36,997 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-09T23:37:37,063 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:37:37,152 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNext/3b115f6a99189b4ae475ada1bbdcd290/.tmp/testFamily/e37ef69477ab4b56a59808087b3d0ae4 2024-12-09T23:37:37,159 INFO [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/.tmp/table/431ab100aec1406191d5766372e0761f 2024-12-09T23:37:37,163 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNext/3b115f6a99189b4ae475ada1bbdcd290/.tmp/testFamily/e37ef69477ab4b56a59808087b3d0ae4 as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNext/3b115f6a99189b4ae475ada1bbdcd290/testFamily/e37ef69477ab4b56a59808087b3d0ae4 2024-12-09T23:37:37,168 DEBUG [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/.tmp/info/f2365a4d7e704414ac2da3b1fae26549 as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/info/f2365a4d7e704414ac2da3b1fae26549 2024-12-09T23:37:37,172 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNext/3b115f6a99189b4ae475ada1bbdcd290/testFamily/e37ef69477ab4b56a59808087b3d0ae4, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T23:37:37,173 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3b115f6a99189b4ae475ada1bbdcd290 in 447ms, sequenceid=8, compaction requested=false 2024-12-09T23:37:37,178 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNext/3b115f6a99189b4ae475ada1bbdcd290/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T23:37:37,179 INFO [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/info/f2365a4d7e704414ac2da3b1fae26549, entries=100, sequenceid=56, filesize=20.1 K 2024-12-09T23:37:37,179 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. 2024-12-09T23:37:37,179 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3b115f6a99189b4ae475ada1bbdcd290: Waiting for close lock at 1733787456725Running coprocessor pre-close hooks at 1733787456725Disabling compacts and flushes for region at 1733787456725Disabling writes for close at 1733787456726 (+1 ms)Obtaining lock to block concurrent updates at 1733787456726Preparing flush snapshotting stores in 3b115f6a99189b4ae475ada1bbdcd290 at 1733787456726Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733787456726Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. at 1733787456727 (+1 ms)Flushing 3b115f6a99189b4ae475ada1bbdcd290/testFamily: creating writer at 1733787456727Flushing 3b115f6a99189b4ae475ada1bbdcd290/testFamily: appending metadata at 1733787456744 (+17 ms)Flushing 3b115f6a99189b4ae475ada1bbdcd290/testFamily: closing flushed file at 1733787456744Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a60b86: reopening flushed file at 1733787457162 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3b115f6a99189b4ae475ada1bbdcd290 in 447ms, sequenceid=8, compaction requested=false at 1733787457173 (+11 ms)Writing region close event to WAL at 1733787457174 (+1 ms)Running coprocessor post-close hooks at 1733787457179 (+5 ms)Closed at 1733787457179 2024-12-09T23:37:37,179 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733787441296.3b115f6a99189b4ae475ada1bbdcd290. 2024-12-09T23:37:37,179 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 71f1f612d5c5e6c68b902d1c7f2e431a, disabling compactions & flushes 2024-12-09T23:37:37,179 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. 2024-12-09T23:37:37,179 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. 2024-12-09T23:37:37,179 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. after waiting 0 ms 2024-12-09T23:37:37,180 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. 2024-12-09T23:37:37,180 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 71f1f612d5c5e6c68b902d1c7f2e431a 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T23:37:37,181 DEBUG [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/.tmp/ns/c34d0638e82b4245a7c8d5ddcbfa5f43 as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/ns/c34d0638e82b4245a7c8d5ddcbfa5f43 2024-12-09T23:37:37,186 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 1588230740, 38a2db52d56427523d5f9315e91a5439, 710180ad3e423dc7415481b84b6d232e, 71f1f612d5c5e6c68b902d1c7f2e431a, 929bfbb576e7b4aa7d44b12e4d0d350f, 9fe7e297263a8471832fe5e82d32a71f, a7a609a0f8fdf69b8dc548a311c82eb2, ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:37:37,188 INFO [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/ns/c34d0638e82b4245a7c8d5ddcbfa5f43, entries=2, sequenceid=56, filesize=5.0 K 2024-12-09T23:37:37,190 DEBUG [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/.tmp/table/431ab100aec1406191d5766372e0761f as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/table/431ab100aec1406191d5766372e0761f 2024-12-09T23:37:37,196 INFO [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/table/431ab100aec1406191d5766372e0761f, entries=20, sequenceid=56, filesize=6.5 K 2024-12-09T23:37:37,198 INFO [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 612ms, sequenceid=56, compaction requested=false 2024-12-09T23:37:37,202 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/71f1f612d5c5e6c68b902d1c7f2e431a/.tmp/testFamily/678ddecf9f4443c685bcd8338347d92b is 61, key is row-0/testFamily:testQualifier/1733787392476/Put/seqid=0 2024-12-09T23:37:37,203 DEBUG [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-09T23:37:37,204 DEBUG [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T23:37:37,205 INFO [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T23:37:37,205 DEBUG [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733787456584Running coprocessor pre-close hooks at 1733787456584Disabling compacts and flushes for region at 1733787456584Disabling writes for close at 1733787456585 (+1 ms)Obtaining lock to block concurrent updates at 1733787456588 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1733787456588Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733787456594 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733787456595 (+1 ms)Flushing 1588230740/info: creating writer at 1733787456597 (+2 ms)Flushing 1588230740/info: appending metadata at 1733787456644 (+47 ms)Flushing 1588230740/info: closing flushed file at 1733787456644Flushing 1588230740/ns: creating writer at 1733787456696 (+52 ms)Flushing 1588230740/ns: appending metadata at 1733787456716 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1733787456716Flushing 1588230740/table: creating writer at 1733787456733 (+17 ms)Flushing 1588230740/table: appending metadata at 1733787456747 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733787456747Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60332586: reopening flushed file at 1733787457167 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25ccfd0d: reopening flushed file at 1733787457179 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78e0c898: reopening flushed file at 1733787457188 (+9 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 612ms, sequenceid=56, compaction requested=false at 1733787457198 (+10 ms)Writing region close event to WAL at 1733787457199 (+1 ms)Running coprocessor post-close hooks at 1733787457204 (+5 ms)Closed at 1733787457204 2024-12-09T23:37:37,205 DEBUG [RS_CLOSE_META-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T23:37:37,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741860_1036 (size=5049) 2024-12-09T23:37:37,386 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 38a2db52d56427523d5f9315e91a5439, 710180ad3e423dc7415481b84b6d232e, 71f1f612d5c5e6c68b902d1c7f2e431a, 929bfbb576e7b4aa7d44b12e4d0d350f, 9fe7e297263a8471832fe5e82d32a71f, a7a609a0f8fdf69b8dc548a311c82eb2, ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:37:37,493 INFO [regionserver/aab77d931c0c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T23:37:37,493 INFO [regionserver/aab77d931c0c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T23:37:37,586 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1321): Waiting on 8 regions to close 2024-12-09T23:37:37,587 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1325): Online Regions={71f1f612d5c5e6c68b902d1c7f2e431a=testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a., 38a2db52d56427523d5f9315e91a5439=testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439., 9fe7e297263a8471832fe5e82d32a71f=testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f., ce42a5d3c00cbd095e73d10231645ad4=hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4., 00225674fa19c083605717af092c94b7=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7., a7a609a0f8fdf69b8dc548a311c82eb2=hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2., 929bfbb576e7b4aa7d44b12e4d0d350f=testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., 710180ad3e423dc7415481b84b6d232e=testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e.} 2024-12-09T23:37:37,587 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 38a2db52d56427523d5f9315e91a5439, 710180ad3e423dc7415481b84b6d232e, 71f1f612d5c5e6c68b902d1c7f2e431a, 929bfbb576e7b4aa7d44b12e4d0d350f, 9fe7e297263a8471832fe5e82d32a71f, a7a609a0f8fdf69b8dc548a311c82eb2, ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:37:37,610 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/71f1f612d5c5e6c68b902d1c7f2e431a/.tmp/testFamily/678ddecf9f4443c685bcd8338347d92b 2024-12-09T23:37:37,622 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/71f1f612d5c5e6c68b902d1c7f2e431a/.tmp/testFamily/678ddecf9f4443c685bcd8338347d92b as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/71f1f612d5c5e6c68b902d1c7f2e431a/testFamily/678ddecf9f4443c685bcd8338347d92b 2024-12-09T23:37:37,631 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/71f1f612d5c5e6c68b902d1c7f2e431a/testFamily/678ddecf9f4443c685bcd8338347d92b, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T23:37:37,632 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 71f1f612d5c5e6c68b902d1c7f2e431a in 452ms, sequenceid=8, compaction requested=false 2024-12-09T23:37:37,639 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/71f1f612d5c5e6c68b902d1c7f2e431a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T23:37:37,639 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. 2024-12-09T23:37:37,640 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 71f1f612d5c5e6c68b902d1c7f2e431a: Waiting for close lock at 1733787457179Running coprocessor pre-close hooks at 1733787457179Disabling compacts and flushes for region at 1733787457179Disabling writes for close at 1733787457180 (+1 ms)Obtaining lock to block concurrent updates at 1733787457180Preparing flush snapshotting stores in 71f1f612d5c5e6c68b902d1c7f2e431a at 1733787457180Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733787457180Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. at 1733787457181 (+1 ms)Flushing 71f1f612d5c5e6c68b902d1c7f2e431a/testFamily: creating writer at 1733787457181Flushing 71f1f612d5c5e6c68b902d1c7f2e431a/testFamily: appending metadata at 1733787457202 (+21 ms)Flushing 71f1f612d5c5e6c68b902d1c7f2e431a/testFamily: closing flushed file at 1733787457202Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fc13621: reopening flushed file at 1733787457621 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 71f1f612d5c5e6c68b902d1c7f2e431a in 452ms, sequenceid=8, compaction requested=false at 1733787457633 (+12 ms)Writing region close event to WAL at 1733787457633Running coprocessor post-close hooks at 1733787457639 (+6 ms)Closed at 1733787457639 2024-12-09T23:37:37,640 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733787389363.71f1f612d5c5e6c68b902d1c7f2e431a. 2024-12-09T23:37:37,640 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 38a2db52d56427523d5f9315e91a5439, disabling compactions & flushes 2024-12-09T23:37:37,640 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. 2024-12-09T23:37:37,640 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. 2024-12-09T23:37:37,640 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. after waiting 0 ms 2024-12-09T23:37:37,640 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. 2024-12-09T23:37:37,640 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 38a2db52d56427523d5f9315e91a5439 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T23:37:37,656 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextException/38a2db52d56427523d5f9315e91a5439/.tmp/testFamily/be9070bc19a54a8ab67aae5924a9ab51 is 61, key is row-0/testFamily:testQualifier/1733787413824/Put/seqid=0 2024-12-09T23:37:37,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741861_1037 (size=5049) 2024-12-09T23:37:37,787 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 38a2db52d56427523d5f9315e91a5439, 710180ad3e423dc7415481b84b6d232e, 929bfbb576e7b4aa7d44b12e4d0d350f, 9fe7e297263a8471832fe5e82d32a71f, a7a609a0f8fdf69b8dc548a311c82eb2, ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:37:37,988 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 38a2db52d56427523d5f9315e91a5439, 710180ad3e423dc7415481b84b6d232e, 929bfbb576e7b4aa7d44b12e4d0d350f, 9fe7e297263a8471832fe5e82d32a71f, a7a609a0f8fdf69b8dc548a311c82eb2, ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:37:38,124 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextException/38a2db52d56427523d5f9315e91a5439/.tmp/testFamily/be9070bc19a54a8ab67aae5924a9ab51 2024-12-09T23:37:38,137 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextException/38a2db52d56427523d5f9315e91a5439/.tmp/testFamily/be9070bc19a54a8ab67aae5924a9ab51 as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextException/38a2db52d56427523d5f9315e91a5439/testFamily/be9070bc19a54a8ab67aae5924a9ab51 2024-12-09T23:37:38,145 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextException/38a2db52d56427523d5f9315e91a5439/testFamily/be9070bc19a54a8ab67aae5924a9ab51, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T23:37:38,146 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 38a2db52d56427523d5f9315e91a5439 in 506ms, sequenceid=8, compaction requested=false 2024-12-09T23:37:38,150 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testRetryOutOfOrderScannerNextException/38a2db52d56427523d5f9315e91a5439/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T23:37:38,151 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. 2024-12-09T23:37:38,151 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 38a2db52d56427523d5f9315e91a5439: Waiting for close lock at 1733787457640Running coprocessor pre-close hooks at 1733787457640Disabling compacts and flushes for region at 1733787457640Disabling writes for close at 1733787457640Obtaining lock to block concurrent updates at 1733787457640Preparing flush snapshotting stores in 38a2db52d56427523d5f9315e91a5439 at 1733787457640Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733787457641 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. at 1733787457641Flushing 38a2db52d56427523d5f9315e91a5439/testFamily: creating writer at 1733787457642 (+1 ms)Flushing 38a2db52d56427523d5f9315e91a5439/testFamily: appending metadata at 1733787457656 (+14 ms)Flushing 38a2db52d56427523d5f9315e91a5439/testFamily: closing flushed file at 1733787457656Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c2cc046: reopening flushed file at 1733787458135 (+479 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 38a2db52d56427523d5f9315e91a5439 in 506ms, sequenceid=8, compaction requested=false at 1733787458146 (+11 ms)Writing region close event to WAL at 1733787458146Running coprocessor post-close hooks at 1733787458151 (+5 ms)Closed at 1733787458151 2024-12-09T23:37:38,151 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733787410773.38a2db52d56427523d5f9315e91a5439. 2024-12-09T23:37:38,152 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9fe7e297263a8471832fe5e82d32a71f, disabling compactions & flushes 2024-12-09T23:37:38,152 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. 2024-12-09T23:37:38,152 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. 2024-12-09T23:37:38,152 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. after waiting 0 ms 2024-12-09T23:37:38,152 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. 2024-12-09T23:37:38,152 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9fe7e297263a8471832fe5e82d32a71f 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T23:37:38,167 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNextAsync/9fe7e297263a8471832fe5e82d32a71f/.tmp/testFamily/4577d501bc0842c7b36fa458750fd7c2 is 61, key is row-0/testFamily:testQualifier/1733787423042/Put/seqid=0 2024-12-09T23:37:38,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741862_1038 (size=5049) 2024-12-09T23:37:38,188 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 710180ad3e423dc7415481b84b6d232e, 929bfbb576e7b4aa7d44b12e4d0d350f, 9fe7e297263a8471832fe5e82d32a71f, a7a609a0f8fdf69b8dc548a311c82eb2, ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:37:38,389 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 710180ad3e423dc7415481b84b6d232e, 929bfbb576e7b4aa7d44b12e4d0d350f, 9fe7e297263a8471832fe5e82d32a71f, a7a609a0f8fdf69b8dc548a311c82eb2, ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:37:38,574 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNextAsync/9fe7e297263a8471832fe5e82d32a71f/.tmp/testFamily/4577d501bc0842c7b36fa458750fd7c2 2024-12-09T23:37:38,589 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-12-09T23:37:38,589 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNextAsync/9fe7e297263a8471832fe5e82d32a71f/.tmp/testFamily/4577d501bc0842c7b36fa458750fd7c2 as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNextAsync/9fe7e297263a8471832fe5e82d32a71f/testFamily/4577d501bc0842c7b36fa458750fd7c2 2024-12-09T23:37:38,589 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1325): Online Regions={9fe7e297263a8471832fe5e82d32a71f=testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f., ce42a5d3c00cbd095e73d10231645ad4=hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4., 00225674fa19c083605717af092c94b7=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7., a7a609a0f8fdf69b8dc548a311c82eb2=hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2., 929bfbb576e7b4aa7d44b12e4d0d350f=testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., 710180ad3e423dc7415481b84b6d232e=testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e.} 2024-12-09T23:37:38,589 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 710180ad3e423dc7415481b84b6d232e, 929bfbb576e7b4aa7d44b12e4d0d350f, 9fe7e297263a8471832fe5e82d32a71f, a7a609a0f8fdf69b8dc548a311c82eb2, ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:37:38,596 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNextAsync/9fe7e297263a8471832fe5e82d32a71f/testFamily/4577d501bc0842c7b36fa458750fd7c2, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T23:37:38,597 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9fe7e297263a8471832fe5e82d32a71f in 445ms, sequenceid=8, compaction requested=false 2024-12-09T23:37:38,601 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnNextAsync/9fe7e297263a8471832fe5e82d32a71f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T23:37:38,602 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. 2024-12-09T23:37:38,602 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9fe7e297263a8471832fe5e82d32a71f: Waiting for close lock at 1733787458151Running coprocessor pre-close hooks at 1733787458151Disabling compacts and flushes for region at 1733787458151Disabling writes for close at 1733787458152 (+1 ms)Obtaining lock to block concurrent updates at 1733787458152Preparing flush snapshotting stores in 9fe7e297263a8471832fe5e82d32a71f at 1733787458152Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733787458152Flushing stores of testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. at 1733787458153 (+1 ms)Flushing 9fe7e297263a8471832fe5e82d32a71f/testFamily: creating writer at 1733787458153Flushing 9fe7e297263a8471832fe5e82d32a71f/testFamily: appending metadata at 1733787458167 (+14 ms)Flushing 9fe7e297263a8471832fe5e82d32a71f/testFamily: closing flushed file at 1733787458167Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58b4c60c: reopening flushed file at 1733787458588 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9fe7e297263a8471832fe5e82d32a71f in 445ms, sequenceid=8, compaction requested=false at 1733787458597 (+9 ms)Writing region close event to WAL at 1733787458597Running coprocessor post-close hooks at 1733787458602 (+5 ms)Closed at 1733787458602 2024-12-09T23:37:38,602 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733787419983.9fe7e297263a8471832fe5e82d32a71f. 2024-12-09T23:37:38,603 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ce42a5d3c00cbd095e73d10231645ad4, disabling compactions & flushes 2024-12-09T23:37:38,603 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. 2024-12-09T23:37:38,603 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. 2024-12-09T23:37:38,603 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. after waiting 0 ms 2024-12-09T23:37:38,603 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. 2024-12-09T23:37:38,603 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ce42a5d3c00cbd095e73d10231645ad4 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T23:37:38,617 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScanner/ce42a5d3c00cbd095e73d10231645ad4/.tmp/testFamily/cc8dbd9837934ed58c4087bfed3a823d is 61, key is row-0/testFamily:testQualifier/1733787395613/Put/seqid=0 2024-12-09T23:37:38,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741863_1039 (size=5049) 2024-12-09T23:37:38,789 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 710180ad3e423dc7415481b84b6d232e, 929bfbb576e7b4aa7d44b12e4d0d350f, a7a609a0f8fdf69b8dc548a311c82eb2, ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:37:38,990 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 710180ad3e423dc7415481b84b6d232e, 929bfbb576e7b4aa7d44b12e4d0d350f, a7a609a0f8fdf69b8dc548a311c82eb2, ce42a5d3c00cbd095e73d10231645ad4 2024-12-09T23:37:39,025 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScanner/ce42a5d3c00cbd095e73d10231645ad4/.tmp/testFamily/cc8dbd9837934ed58c4087bfed3a823d 2024-12-09T23:37:39,043 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScanner/ce42a5d3c00cbd095e73d10231645ad4/.tmp/testFamily/cc8dbd9837934ed58c4087bfed3a823d as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScanner/ce42a5d3c00cbd095e73d10231645ad4/testFamily/cc8dbd9837934ed58c4087bfed3a823d 2024-12-09T23:37:39,051 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScanner/ce42a5d3c00cbd095e73d10231645ad4/testFamily/cc8dbd9837934ed58c4087bfed3a823d, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T23:37:39,052 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ce42a5d3c00cbd095e73d10231645ad4 in 449ms, sequenceid=8, compaction requested=false 2024-12-09T23:37:39,057 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScanner/ce42a5d3c00cbd095e73d10231645ad4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T23:37:39,057 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. 2024-12-09T23:37:39,058 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ce42a5d3c00cbd095e73d10231645ad4: Waiting for close lock at 1733787458602Running coprocessor pre-close hooks at 1733787458602Disabling compacts and flushes for region at 1733787458602Disabling writes for close at 1733787458603 (+1 ms)Obtaining lock to block concurrent updates at 1733787458603Preparing flush snapshotting stores in ce42a5d3c00cbd095e73d10231645ad4 at 1733787458603Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733787458603Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. at 1733787458604 (+1 ms)Flushing ce42a5d3c00cbd095e73d10231645ad4/testFamily: creating writer at 1733787458604Flushing ce42a5d3c00cbd095e73d10231645ad4/testFamily: appending metadata at 1733787458617 (+13 ms)Flushing ce42a5d3c00cbd095e73d10231645ad4/testFamily: closing flushed file at 1733787458617Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dbb2ba6: reopening flushed file at 1733787459042 (+425 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ce42a5d3c00cbd095e73d10231645ad4 in 449ms, sequenceid=8, compaction requested=false at 1733787459052 (+10 ms)Writing region close event to WAL at 1733787459053 (+1 ms)Running coprocessor post-close hooks at 1733787459057 (+4 ms)Closed at 1733787459057 2024-12-09T23:37:39,058 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733787392566.ce42a5d3c00cbd095e73d10231645ad4. 2024-12-09T23:37:39,058 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 00225674fa19c083605717af092c94b7, disabling compactions & flushes 2024-12-09T23:37:39,058 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. 2024-12-09T23:37:39,058 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. 2024-12-09T23:37:39,058 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. after waiting 0 ms 2024-12-09T23:37:39,058 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. 2024-12-09T23:37:39,058 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 00225674fa19c083605717af092c94b7 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T23:37:39,073 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/00225674fa19c083605717af092c94b7/.tmp/testFamily/b3f36551bf304f47876be6152a8824b2 is 61, key is row-0/testFamily:testQualifier/1733787435252/Put/seqid=0 2024-12-09T23:37:39,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741864_1040 (size=5049) 2024-12-09T23:37:39,190 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 710180ad3e423dc7415481b84b6d232e, 929bfbb576e7b4aa7d44b12e4d0d350f, a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:37:39,391 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 00225674fa19c083605717af092c94b7, 710180ad3e423dc7415481b84b6d232e, 929bfbb576e7b4aa7d44b12e4d0d350f, a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:37:39,480 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/00225674fa19c083605717af092c94b7/.tmp/testFamily/b3f36551bf304f47876be6152a8824b2 2024-12-09T23:37:39,495 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/00225674fa19c083605717af092c94b7/.tmp/testFamily/b3f36551bf304f47876be6152a8824b2 as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/00225674fa19c083605717af092c94b7/testFamily/b3f36551bf304f47876be6152a8824b2 2024-12-09T23:37:39,502 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/00225674fa19c083605717af092c94b7/testFamily/b3f36551bf304f47876be6152a8824b2, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T23:37:39,504 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 00225674fa19c083605717af092c94b7 in 446ms, sequenceid=8, compaction requested=false 2024-12-09T23:37:39,508 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/00225674fa19c083605717af092c94b7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T23:37:39,509 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. 2024-12-09T23:37:39,509 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 00225674fa19c083605717af092c94b7: Waiting for close lock at 1733787459058Running coprocessor pre-close hooks at 1733787459058Disabling compacts and flushes for region at 1733787459058Disabling writes for close at 1733787459058Obtaining lock to block concurrent updates at 1733787459058Preparing flush snapshotting stores in 00225674fa19c083605717af092c94b7 at 1733787459058Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733787459058Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. at 1733787459059 (+1 ms)Flushing 00225674fa19c083605717af092c94b7/testFamily: creating writer at 1733787459059Flushing 00225674fa19c083605717af092c94b7/testFamily: appending metadata at 1733787459072 (+13 ms)Flushing 00225674fa19c083605717af092c94b7/testFamily: closing flushed file at 1733787459072Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64c90f69: reopening flushed file at 1733787459494 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 00225674fa19c083605717af092c94b7 in 446ms, sequenceid=8, compaction requested=false at 1733787459504 (+10 ms)Writing region close event to WAL at 1733787459504Running coprocessor post-close hooks at 1733787459509 (+5 ms)Closed at 1733787459509 2024-12-09T23:37:39,509 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733787432193.00225674fa19c083605717af092c94b7. 2024-12-09T23:37:39,510 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a7a609a0f8fdf69b8dc548a311c82eb2, disabling compactions & flushes 2024-12-09T23:37:39,510 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. 2024-12-09T23:37:39,510 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. 2024-12-09T23:37:39,510 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. after waiting 0 ms 2024-12-09T23:37:39,510 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. 2024-12-09T23:37:39,510 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a7a609a0f8fdf69b8dc548a311c82eb2 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T23:37:39,527 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNextAsync/a7a609a0f8fdf69b8dc548a311c82eb2/.tmp/testFamily/3cd5bd5d1b394c29950d8d76ed44bf45 is 61, key is row-0/testFamily:testQualifier/1733787404725/Put/seqid=0 2024-12-09T23:37:39,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741865_1041 (size=5049) 2024-12-09T23:37:39,591 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-09T23:37:39,591 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1325): Online Regions={a7a609a0f8fdf69b8dc548a311c82eb2=hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2., 929bfbb576e7b4aa7d44b12e4d0d350f=testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., 710180ad3e423dc7415481b84b6d232e=testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e.} 2024-12-09T23:37:39,591 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 710180ad3e423dc7415481b84b6d232e, 929bfbb576e7b4aa7d44b12e4d0d350f, a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:37:39,791 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 710180ad3e423dc7415481b84b6d232e, 929bfbb576e7b4aa7d44b12e4d0d350f, a7a609a0f8fdf69b8dc548a311c82eb2 2024-12-09T23:37:39,934 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNextAsync/a7a609a0f8fdf69b8dc548a311c82eb2/.tmp/testFamily/3cd5bd5d1b394c29950d8d76ed44bf45 2024-12-09T23:37:39,948 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNextAsync/a7a609a0f8fdf69b8dc548a311c82eb2/.tmp/testFamily/3cd5bd5d1b394c29950d8d76ed44bf45 as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNextAsync/a7a609a0f8fdf69b8dc548a311c82eb2/testFamily/3cd5bd5d1b394c29950d8d76ed44bf45 2024-12-09T23:37:39,958 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNextAsync/a7a609a0f8fdf69b8dc548a311c82eb2/testFamily/3cd5bd5d1b394c29950d8d76ed44bf45, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T23:37:39,959 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a7a609a0f8fdf69b8dc548a311c82eb2 in 449ms, sequenceid=8, compaction requested=false 2024-12-09T23:37:39,964 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/hbase/testMetaScanTimeoutOnNextAsync/a7a609a0f8fdf69b8dc548a311c82eb2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T23:37:39,964 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. 2024-12-09T23:37:39,964 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a7a609a0f8fdf69b8dc548a311c82eb2: Waiting for close lock at 1733787459509Running coprocessor pre-close hooks at 1733787459509Disabling compacts and flushes for region at 1733787459509Disabling writes for close at 1733787459510 (+1 ms)Obtaining lock to block concurrent updates at 1733787459510Preparing flush snapshotting stores in a7a609a0f8fdf69b8dc548a311c82eb2 at 1733787459510Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733787459510Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. at 1733787459511 (+1 ms)Flushing a7a609a0f8fdf69b8dc548a311c82eb2/testFamily: creating writer at 1733787459511Flushing a7a609a0f8fdf69b8dc548a311c82eb2/testFamily: appending metadata at 1733787459527 (+16 ms)Flushing a7a609a0f8fdf69b8dc548a311c82eb2/testFamily: closing flushed file at 1733787459527Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1acf7580: reopening flushed file at 1733787459946 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a7a609a0f8fdf69b8dc548a311c82eb2 in 449ms, sequenceid=8, compaction requested=false at 1733787459959 (+13 ms)Writing region close event to WAL at 1733787459960 (+1 ms)Running coprocessor post-close hooks at 1733787459964 (+4 ms)Closed at 1733787459964 2024-12-09T23:37:39,965 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733787401660.a7a609a0f8fdf69b8dc548a311c82eb2. 2024-12-09T23:37:39,965 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 929bfbb576e7b4aa7d44b12e4d0d350f, disabling compactions & flushes 2024-12-09T23:37:39,965 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. 2024-12-09T23:37:39,965 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. 2024-12-09T23:37:39,965 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. after waiting 0 ms 2024-12-09T23:37:39,965 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. 2024-12-09T23:37:39,965 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 929bfbb576e7b4aa7d44b12e4d0d350f 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T23:37:39,981 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScannerAsync/929bfbb576e7b4aa7d44b12e4d0d350f/.tmp/testFamily/3ab6af78511f4eb5acf1f7bb34dd3d13 is 61, key is row-0/testFamily:testQualifier/1733787453455/Put/seqid=0 2024-12-09T23:37:39,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741866_1042 (size=5049) 2024-12-09T23:37:39,992 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 710180ad3e423dc7415481b84b6d232e, 929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:40,192 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 710180ad3e423dc7415481b84b6d232e, 929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:40,388 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScannerAsync/929bfbb576e7b4aa7d44b12e4d0d350f/.tmp/testFamily/3ab6af78511f4eb5acf1f7bb34dd3d13 2024-12-09T23:37:40,393 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 710180ad3e423dc7415481b84b6d232e, 929bfbb576e7b4aa7d44b12e4d0d350f 2024-12-09T23:37:40,402 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScannerAsync/929bfbb576e7b4aa7d44b12e4d0d350f/.tmp/testFamily/3ab6af78511f4eb5acf1f7bb34dd3d13 as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScannerAsync/929bfbb576e7b4aa7d44b12e4d0d350f/testFamily/3ab6af78511f4eb5acf1f7bb34dd3d13 2024-12-09T23:37:40,409 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScannerAsync/929bfbb576e7b4aa7d44b12e4d0d350f/testFamily/3ab6af78511f4eb5acf1f7bb34dd3d13, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T23:37:40,410 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 929bfbb576e7b4aa7d44b12e4d0d350f in 445ms, sequenceid=8, compaction requested=false 2024-12-09T23:37:40,415 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScannerAsync/929bfbb576e7b4aa7d44b12e4d0d350f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T23:37:40,416 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. 2024-12-09T23:37:40,416 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 929bfbb576e7b4aa7d44b12e4d0d350f: Waiting for close lock at 1733787459965Running coprocessor pre-close hooks at 1733787459965Disabling compacts and flushes for region at 1733787459965Disabling writes for close at 1733787459965Obtaining lock to block concurrent updates at 1733787459965Preparing flush snapshotting stores in 929bfbb576e7b4aa7d44b12e4d0d350f at 1733787459965Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733787459965Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. at 1733787459966 (+1 ms)Flushing 929bfbb576e7b4aa7d44b12e4d0d350f/testFamily: creating writer at 1733787459966Flushing 929bfbb576e7b4aa7d44b12e4d0d350f/testFamily: appending metadata at 1733787459981 (+15 ms)Flushing 929bfbb576e7b4aa7d44b12e4d0d350f/testFamily: closing flushed file at 1733787459981Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@445ad4a: reopening flushed file at 1733787460400 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 929bfbb576e7b4aa7d44b12e4d0d350f in 445ms, sequenceid=8, compaction requested=false at 1733787460410 (+10 ms)Writing region close event to WAL at 1733787460411 (+1 ms)Running coprocessor post-close hooks at 1733787460416 (+5 ms)Closed at 1733787460416 2024-12-09T23:37:40,416 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733787450392.929bfbb576e7b4aa7d44b12e4d0d350f. 2024-12-09T23:37:40,416 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 710180ad3e423dc7415481b84b6d232e, disabling compactions & flushes 2024-12-09T23:37:40,416 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. 2024-12-09T23:37:40,416 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. 2024-12-09T23:37:40,416 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. after waiting 0 ms 2024-12-09T23:37:40,416 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. 2024-12-09T23:37:40,416 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 710180ad3e423dc7415481b84b6d232e 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T23:37:40,431 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScanner/710180ad3e423dc7415481b84b6d232e/.tmp/testFamily/c7fccc44dd9947ed8e4217c04b02c3ff is 61, key is row-0/testFamily:testQualifier/1733787416943/Put/seqid=0 2024-12-09T23:37:40,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741867_1043 (size=5049) 2024-12-09T23:37:40,593 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T23:37:40,593 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1325): Online Regions={710180ad3e423dc7415481b84b6d232e=testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e.} 2024-12-09T23:37:40,593 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 710180ad3e423dc7415481b84b6d232e 2024-12-09T23:37:40,794 DEBUG [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1351): Waiting on 710180ad3e423dc7415481b84b6d232e 2024-12-09T23:37:40,839 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScanner/710180ad3e423dc7415481b84b6d232e/.tmp/testFamily/c7fccc44dd9947ed8e4217c04b02c3ff 2024-12-09T23:37:40,854 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScanner/710180ad3e423dc7415481b84b6d232e/.tmp/testFamily/c7fccc44dd9947ed8e4217c04b02c3ff as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScanner/710180ad3e423dc7415481b84b6d232e/testFamily/c7fccc44dd9947ed8e4217c04b02c3ff 2024-12-09T23:37:40,861 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScanner/710180ad3e423dc7415481b84b6d232e/testFamily/c7fccc44dd9947ed8e4217c04b02c3ff, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T23:37:40,863 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 710180ad3e423dc7415481b84b6d232e in 447ms, sequenceid=8, compaction requested=false 2024-12-09T23:37:40,868 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/data/default/testNormalScanTimeoutOnOpenScanner/710180ad3e423dc7415481b84b6d232e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T23:37:40,869 INFO [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. 2024-12-09T23:37:40,869 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 710180ad3e423dc7415481b84b6d232e: Waiting for close lock at 1733787460416Running coprocessor pre-close hooks at 1733787460416Disabling compacts and flushes for region at 1733787460416Disabling writes for close at 1733787460416Obtaining lock to block concurrent updates at 1733787460416Preparing flush snapshotting stores in 710180ad3e423dc7415481b84b6d232e at 1733787460416Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733787460416Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. at 1733787460417 (+1 ms)Flushing 710180ad3e423dc7415481b84b6d232e/testFamily: creating writer at 1733787460417Flushing 710180ad3e423dc7415481b84b6d232e/testFamily: appending metadata at 1733787460431 (+14 ms)Flushing 710180ad3e423dc7415481b84b6d232e/testFamily: closing flushed file at 1733787460431Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6754f0d: reopening flushed file at 1733787460852 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 710180ad3e423dc7415481b84b6d232e in 447ms, sequenceid=8, compaction requested=false at 1733787460863 (+11 ms)Writing region close event to WAL at 1733787460863Running coprocessor post-close hooks at 1733787460869 (+6 ms)Closed at 1733787460869 2024-12-09T23:37:40,869 DEBUG [RS_CLOSE_REGION-regionserver/aab77d931c0c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733787413878.710180ad3e423dc7415481b84b6d232e. 2024-12-09T23:37:40,995 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(976): stopping server aab77d931c0c,39071,1733787355928; all regions closed. 2024-12-09T23:37:41,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741834_1010 (size=19479) 2024-12-09T23:37:41,012 DEBUG [RS:0;aab77d931c0c:39071 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/oldWALs 2024-12-09T23:37:41,012 INFO [RS:0;aab77d931c0c:39071 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL aab77d931c0c%2C39071%2C1733787355928.meta:.meta(num 1733787388900) 2024-12-09T23:37:41,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741833_1009 (size=20031) 2024-12-09T23:37:41,016 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/WALs/aab77d931c0c,39071,1733787355928/aab77d931c0c%2C39071%2C1733787355928.1733787388543 not finished, retry = 0 2024-12-09T23:37:41,122 DEBUG [RS:0;aab77d931c0c:39071 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/oldWALs 2024-12-09T23:37:41,122 INFO [RS:0;aab77d931c0c:39071 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL aab77d931c0c%2C39071%2C1733787355928:(num 1733787388543) 2024-12-09T23:37:41,122 DEBUG [RS:0;aab77d931c0c:39071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T23:37:41,122 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T23:37:41,123 INFO [RS:0;aab77d931c0c:39071 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T23:37:41,123 INFO [RS:0;aab77d931c0c:39071 {}] hbase.ChoreService(370): Chore service for: regionserver/aab77d931c0c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T23:37:41,124 INFO [RS:0;aab77d931c0c:39071 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T23:37:41,124 INFO [regionserver/aab77d931c0c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T23:37:41,125 INFO [RS:0;aab77d931c0c:39071 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39071 2024-12-09T23:37:41,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T23:37:41,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/aab77d931c0c,39071,1733787355928 2024-12-09T23:37:41,143 INFO [RS:0;aab77d931c0c:39071 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T23:37:41,154 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [aab77d931c0c,39071,1733787355928] 2024-12-09T23:37:41,164 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/aab77d931c0c,39071,1733787355928 already deleted, retry=false 2024-12-09T23:37:41,165 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; aab77d931c0c,39071,1733787355928 expired; onlineServers=0 2024-12-09T23:37:41,165 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'aab77d931c0c,33463,1733787355045' ***** 2024-12-09T23:37:41,165 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T23:37:41,165 INFO [M:0;aab77d931c0c:33463 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T23:37:41,165 INFO [M:0;aab77d931c0c:33463 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T23:37:41,166 DEBUG [M:0;aab77d931c0c:33463 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T23:37:41,166 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T23:37:41,166 DEBUG [M:0;aab77d931c0c:33463 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T23:37:41,166 DEBUG [master/aab77d931c0c:0:becomeActiveMaster-HFileCleaner.large.0-1733787358346 {}] cleaner.HFileCleaner(306): Exit Thread[master/aab77d931c0c:0:becomeActiveMaster-HFileCleaner.large.0-1733787358346,5,FailOnTimeoutGroup] 2024-12-09T23:37:41,166 DEBUG [master/aab77d931c0c:0:becomeActiveMaster-HFileCleaner.small.0-1733787358347 {}] cleaner.HFileCleaner(306): Exit Thread[master/aab77d931c0c:0:becomeActiveMaster-HFileCleaner.small.0-1733787358347,5,FailOnTimeoutGroup] 2024-12-09T23:37:41,167 INFO [M:0;aab77d931c0c:33463 {}] hbase.ChoreService(370): Chore service for: master/aab77d931c0c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T23:37:41,167 INFO [M:0;aab77d931c0c:33463 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T23:37:41,167 DEBUG [M:0;aab77d931c0c:33463 {}] master.HMaster(1795): Stopping service threads 2024-12-09T23:37:41,167 INFO [M:0;aab77d931c0c:33463 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T23:37:41,168 INFO [M:0;aab77d931c0c:33463 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T23:37:41,169 INFO [M:0;aab77d931c0c:33463 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T23:37:41,169 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T23:37:41,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T23:37:41,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T23:37:41,175 DEBUG [M:0;aab77d931c0c:33463 {}] zookeeper.ZKUtil(347): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T23:37:41,175 WARN [M:0;aab77d931c0c:33463 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T23:37:41,176 INFO [M:0;aab77d931c0c:33463 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/.lastflushedseqids 2024-12-09T23:37:41,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741868_1044 (size=649) 2024-12-09T23:37:41,220 INFO [regionserver/aab77d931c0c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T23:37:41,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T23:37:41,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39071-0x1000cef8b010001, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T23:37:41,255 INFO [RS:0;aab77d931c0c:39071 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T23:37:41,255 INFO [RS:0;aab77d931c0c:39071 {}] regionserver.HRegionServer(1031): Exiting; stopping=aab77d931c0c,39071,1733787355928; zookeeper connection closed. 2024-12-09T23:37:41,256 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@50606ae2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@50606ae2 2024-12-09T23:37:41,256 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-09T23:37:41,592 INFO [M:0;aab77d931c0c:33463 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T23:37:41,593 INFO [M:0;aab77d931c0c:33463 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T23:37:41,594 DEBUG [M:0;aab77d931c0c:33463 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T23:37:41,594 INFO [M:0;aab77d931c0c:33463 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:37:41,594 DEBUG [M:0;aab77d931c0c:33463 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:37:41,594 DEBUG [M:0;aab77d931c0c:33463 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T23:37:41,594 DEBUG [M:0;aab77d931c0c:33463 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:37:41,595 INFO [M:0;aab77d931c0c:33463 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.10 KB heapSize=190.99 KB 2024-12-09T23:37:41,614 DEBUG [M:0;aab77d931c0c:33463 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe3c24d565c84206a76ffaf7311f27c0 is 82, key is hbase:meta,,1/info:regioninfo/1733787388985/Put/seqid=0 2024-12-09T23:37:41,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741869_1045 (size=5672) 2024-12-09T23:37:42,029 INFO [M:0;aab77d931c0c:33463 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe3c24d565c84206a76ffaf7311f27c0 2024-12-09T23:37:42,055 DEBUG [M:0;aab77d931c0c:33463 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0b337388176243768683b0f907920f1c is 813, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733787390667/Put/seqid=0 2024-12-09T23:37:42,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741870_1046 (size=14079) 2024-12-09T23:37:42,228 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T23:37:42,461 INFO [M:0;aab77d931c0c:33463 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0b337388176243768683b0f907920f1c 2024-12-09T23:37:42,474 INFO [M:0;aab77d931c0c:33463 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0b337388176243768683b0f907920f1c 2024-12-09T23:37:42,489 DEBUG [M:0;aab77d931c0c:33463 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b94a73e248dd4c62b109986870fa0d8e is 69, key is aab77d931c0c,39071,1733787355928/rs:state/1733787358418/Put/seqid=0 2024-12-09T23:37:42,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741871_1047 (size=5156) 2024-12-09T23:37:42,896 INFO [M:0;aab77d931c0c:33463 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b94a73e248dd4c62b109986870fa0d8e 2024-12-09T23:37:42,909 DEBUG [M:0;aab77d931c0c:33463 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe3c24d565c84206a76ffaf7311f27c0 as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fe3c24d565c84206a76ffaf7311f27c0 2024-12-09T23:37:42,915 INFO [M:0;aab77d931c0c:33463 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fe3c24d565c84206a76ffaf7311f27c0, entries=8, sequenceid=317, filesize=5.5 K 2024-12-09T23:37:42,917 DEBUG [M:0;aab77d931c0c:33463 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0b337388176243768683b0f907920f1c as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0b337388176243768683b0f907920f1c 2024-12-09T23:37:42,923 INFO [M:0;aab77d931c0c:33463 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0b337388176243768683b0f907920f1c 2024-12-09T23:37:42,923 INFO [M:0;aab77d931c0c:33463 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0b337388176243768683b0f907920f1c, entries=33, sequenceid=317, filesize=13.7 K 2024-12-09T23:37:42,924 DEBUG [M:0;aab77d931c0c:33463 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b94a73e248dd4c62b109986870fa0d8e as hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b94a73e248dd4c62b109986870fa0d8e 2024-12-09T23:37:42,930 INFO [M:0;aab77d931c0c:33463 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37497/user/jenkins/test-data/a2e9f592-b52e-11cb-b551-21c48342005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b94a73e248dd4c62b109986870fa0d8e, entries=1, sequenceid=317, filesize=5.0 K 2024-12-09T23:37:42,932 INFO [M:0;aab77d931c0c:33463 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.10 KB/167011, heapSize ~190.70 KB/195272, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1337ms, sequenceid=317, compaction requested=false 2024-12-09T23:37:42,933 INFO [M:0;aab77d931c0c:33463 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T23:37:42,933 DEBUG [M:0;aab77d931c0c:33463 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733787461594Disabling compacts and flushes for region at 1733787461594Disabling writes for close at 1733787461594Obtaining lock to block concurrent updates at 1733787461595 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733787461595Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=167011, getHeapSize=195512, getOffHeapSize=0, getCellsCount=364 at 1733787461596 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733787461597 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733787461597Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733787461614 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733787461614Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733787462040 (+426 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733787462054 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733787462054Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733787462474 (+420 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733787462489 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733787462489Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5abad550: reopening flushed file at 1733787462908 (+419 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38d20d7: reopening flushed file at 1733787462915 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47f2e928: reopening flushed file at 1733787462923 (+8 ms)Finished flush of dataSize ~163.10 KB/167011, heapSize ~190.70 KB/195272, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1337ms, sequenceid=317, compaction requested=false at 1733787462932 (+9 ms)Writing region close event to WAL at 1733787462933 (+1 ms)Closed at 1733787462933 2024-12-09T23:37:42,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33481 is added to blk_1073741830_1006 (size=188223) 2024-12-09T23:37:42,938 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T23:37:42,938 INFO [M:0;aab77d931c0c:33463 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T23:37:42,938 INFO [M:0;aab77d931c0c:33463 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33463 2024-12-09T23:37:42,938 INFO [M:0;aab77d931c0c:33463 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T23:37:43,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T23:37:43,085 INFO [M:0;aab77d931c0c:33463 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T23:37:43,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33463-0x1000cef8b010000, quorum=127.0.0.1:52508, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T23:37:43,180 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T23:37:43,185 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T23:37:43,186 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T23:37:43,186 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T23:37:43,187 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/hadoop.log.dir/,STOPPED} 2024-12-09T23:37:43,190 WARN [BP-1725909415-172.17.0.3-1733787351209 heartbeating to localhost/127.0.0.1:37497 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T23:37:43,190 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T23:37:43,190 WARN [BP-1725909415-172.17.0.3-1733787351209 heartbeating to localhost/127.0.0.1:37497 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1725909415-172.17.0.3-1733787351209 (Datanode Uuid 4dd69fc6-a6f0-4e54-9246-b2e1e64836c1) service to localhost/127.0.0.1:37497 2024-12-09T23:37:43,190 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T23:37:43,191 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/cluster_9d583092-22a2-df5c-e9ea-fff7e629db2e/data/data1/current/BP-1725909415-172.17.0.3-1733787351209 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T23:37:43,191 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/cluster_9d583092-22a2-df5c-e9ea-fff7e629db2e/data/data2/current/BP-1725909415-172.17.0.3-1733787351209 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T23:37:43,192 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T23:37:43,198 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T23:37:43,199 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T23:37:43,199 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T23:37:43,199 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T23:37:43,199 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/bb68c671-39ff-0b01-dc79-0f905253d1d3/hadoop.log.dir/,STOPPED} 2024-12-09T23:37:43,205 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T23:37:43,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down