2024-11-25 07:57:12,620 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-25 07:57:12,638 main DEBUG Took 0.016121 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 07:57:12,638 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 07:57:12,639 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 07:57:12,640 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 07:57:12,641 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:57:12,648 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 07:57:12,663 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:57:12,664 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:57:12,665 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:57:12,665 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:57:12,666 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:57:12,666 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:57:12,668 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:57:12,668 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:57:12,669 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:57:12,669 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:57:12,670 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:57:12,671 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:57:12,671 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:57:12,672 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:57:12,672 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:57:12,673 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:57:12,673 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:57:12,674 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:57:12,674 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:57:12,675 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:57:12,675 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:57:12,676 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:57:12,676 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:57:12,676 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 07:57:12,677 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:57:12,677 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 07:57:12,679 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 07:57:12,680 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 07:57:12,682 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 07:57:12,683 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 07:57:12,684 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 07:57:12,684 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 07:57:12,693 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 07:57:12,696 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 07:57:12,697 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 07:57:12,698 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 07:57:12,698 main DEBUG createAppenders(={Console}) 2024-11-25 07:57:12,699 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-25 07:57:12,699 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-25 07:57:12,700 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-25 07:57:12,700 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 07:57:12,700 main DEBUG OutputStream closed 2024-11-25 07:57:12,701 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 07:57:12,701 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 07:57:12,701 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-25 07:57:12,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 07:57:12,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 07:57:12,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 07:57:12,790 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 07:57:12,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 07:57:12,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 07:57:12,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 07:57:12,792 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 07:57:12,792 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 07:57:12,792 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 07:57:12,793 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 07:57:12,793 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 07:57:12,794 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 07:57:12,794 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 07:57:12,794 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 07:57:12,795 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 07:57:12,795 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 07:57:12,796 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 07:57:12,799 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 07:57:12,799 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-25 07:57:12,799 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 07:57:12,800 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-25T07:57:12,815 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-25 07:57:12,818 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 07:57:12,819 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T07:57:13,089 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e 2024-11-25T07:57:13,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T07:57:13,127 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/cluster_ae7675c4-28d6-36d0-1e5b-73429a44c766, deleteOnExit=true 2024-11-25T07:57:13,128 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T07:57:13,129 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/test.cache.data in system properties and HBase conf 2024-11-25T07:57:13,130 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T07:57:13,131 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/hadoop.log.dir in system properties and HBase conf 2024-11-25T07:57:13,132 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T07:57:13,133 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T07:57:13,133 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T07:57:13,235 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T07:57:13,355 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T07:57:13,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T07:57:13,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T07:57:13,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T07:57:13,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T07:57:13,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T07:57:13,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T07:57:13,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T07:57:13,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T07:57:13,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T07:57:13,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/nfs.dump.dir in system properties and HBase conf 2024-11-25T07:57:13,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/java.io.tmpdir in system properties and HBase conf 2024-11-25T07:57:13,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T07:57:13,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T07:57:13,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T07:57:14,526 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T07:57:14,615 INFO [Time-limited test {}] log.Log(170): Logging initialized @2782ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T07:57:14,702 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T07:57:14,770 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T07:57:14,797 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T07:57:14,798 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T07:57:14,800 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T07:57:14,813 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T07:57:14,817 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/hadoop.log.dir/,AVAILABLE} 2024-11-25T07:57:14,818 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T07:57:15,020 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@40b93e99{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/java.io.tmpdir/jetty-localhost-46445-hadoop-hdfs-3_4_1-tests_jar-_-any-17155536341965005455/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T07:57:15,031 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:46445} 2024-11-25T07:57:15,031 INFO [Time-limited test {}] server.Server(415): Started @3199ms 2024-11-25T07:57:15,649 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T07:57:15,659 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T07:57:15,660 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T07:57:15,660 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T07:57:15,660 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T07:57:15,661 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/hadoop.log.dir/,AVAILABLE} 2024-11-25T07:57:15,662 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T07:57:15,794 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@429b6cf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/java.io.tmpdir/jetty-localhost-45283-hadoop-hdfs-3_4_1-tests_jar-_-any-9589015316723943131/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T07:57:15,795 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:45283} 2024-11-25T07:57:15,795 INFO [Time-limited test {}] server.Server(415): Started @3963ms 2024-11-25T07:57:15,855 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T07:57:17,135 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/cluster_ae7675c4-28d6-36d0-1e5b-73429a44c766/data/data2/current/BP-1047878032-172.17.0.2-1732521433990/current, will proceed with Du for space computation calculation, 2024-11-25T07:57:17,135 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/cluster_ae7675c4-28d6-36d0-1e5b-73429a44c766/data/data1/current/BP-1047878032-172.17.0.2-1732521433990/current, will proceed with Du for space computation calculation, 2024-11-25T07:57:17,170 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T07:57:17,223 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7729eb1479a1f65a with lease ID 0x1e7041fa9a748a3e: Processing first storage report for DS-9e5bcbd9-660b-4360-9579-f91c2a9d7df5 from datanode DatanodeRegistration(127.0.0.1:32909, datanodeUuid=b3316b2a-35b9-482a-9f92-d7e9b3114fac, infoPort=34577, infoSecurePort=0, ipcPort=44365, storageInfo=lv=-57;cid=testClusterID;nsid=170659208;c=1732521433990) 2024-11-25T07:57:17,224 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7729eb1479a1f65a with lease ID 0x1e7041fa9a748a3e: from storage DS-9e5bcbd9-660b-4360-9579-f91c2a9d7df5 node DatanodeRegistration(127.0.0.1:32909, datanodeUuid=b3316b2a-35b9-482a-9f92-d7e9b3114fac, infoPort=34577, infoSecurePort=0, ipcPort=44365, storageInfo=lv=-57;cid=testClusterID;nsid=170659208;c=1732521433990), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-25T07:57:17,225 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7729eb1479a1f65a with lease ID 0x1e7041fa9a748a3e: Processing first storage report for DS-d1f34dcd-5a41-45ad-90a1-c6a6571ea0e1 from datanode DatanodeRegistration(127.0.0.1:32909, datanodeUuid=b3316b2a-35b9-482a-9f92-d7e9b3114fac, infoPort=34577, infoSecurePort=0, ipcPort=44365, storageInfo=lv=-57;cid=testClusterID;nsid=170659208;c=1732521433990) 2024-11-25T07:57:17,225 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7729eb1479a1f65a with lease ID 0x1e7041fa9a748a3e: from storage DS-d1f34dcd-5a41-45ad-90a1-c6a6571ea0e1 node DatanodeRegistration(127.0.0.1:32909, datanodeUuid=b3316b2a-35b9-482a-9f92-d7e9b3114fac, infoPort=34577, infoSecurePort=0, ipcPort=44365, storageInfo=lv=-57;cid=testClusterID;nsid=170659208;c=1732521433990), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-25T07:57:17,309 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e 2024-11-25T07:57:17,385 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/cluster_ae7675c4-28d6-36d0-1e5b-73429a44c766/zookeeper_0, clientPort=53620, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/cluster_ae7675c4-28d6-36d0-1e5b-73429a44c766/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/cluster_ae7675c4-28d6-36d0-1e5b-73429a44c766/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T07:57:17,396 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53620 2024-11-25T07:57:17,411 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:57:17,415 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:57:17,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741825_1001 (size=7) 2024-11-25T07:57:18,055 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574 with version=8 2024-11-25T07:57:18,056 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/hbase-staging 2024-11-25T07:57:18,158 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T07:57:18,397 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7fb8763a5d32:0 server-side Connection retries=9 2024-11-25T07:57:18,406 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T07:57:18,407 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T07:57:18,411 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T07:57:18,411 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T07:57:18,411 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T07:57:18,583 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T07:57:18,653 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T07:57:18,665 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T07:57:18,670 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T07:57:18,703 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 76273 (auto-detected) 2024-11-25T07:57:18,705 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-25T07:57:18,731 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35737 2024-11-25T07:57:18,758 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35737 connecting to ZooKeeper ensemble=127.0.0.1:53620 2024-11-25T07:57:18,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:357370x0, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T07:57:18,910 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35737-0x10170ff24600000 connected 2024-11-25T07:57:19,034 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:57:19,037 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:57:19,051 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T07:57:19,056 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574, hbase.cluster.distributed=false 2024-11-25T07:57:19,085 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T07:57:19,092 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35737 2024-11-25T07:57:19,093 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35737 2024-11-25T07:57:19,093 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35737 2024-11-25T07:57:19,094 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35737 2024-11-25T07:57:19,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35737 2024-11-25T07:57:19,238 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7fb8763a5d32:0 server-side Connection retries=9 2024-11-25T07:57:19,240 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T07:57:19,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T07:57:19,241 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T07:57:19,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T07:57:19,242 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T07:57:19,246 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T07:57:19,249 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T07:57:19,250 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41809 2024-11-25T07:57:19,253 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41809 connecting to ZooKeeper ensemble=127.0.0.1:53620 2024-11-25T07:57:19,254 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:57:19,258 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:57:19,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:418090x0, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T07:57:19,275 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41809-0x10170ff24600001 connected 2024-11-25T07:57:19,275 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T07:57:19,280 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T07:57:19,289 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T07:57:19,292 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T07:57:19,298 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T07:57:19,299 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41809 2024-11-25T07:57:19,299 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41809 2024-11-25T07:57:19,300 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41809 2024-11-25T07:57:19,308 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41809 2024-11-25T07:57:19,309 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41809 2024-11-25T07:57:19,325 DEBUG [M:0;7fb8763a5d32:35737 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7fb8763a5d32:35737 2024-11-25T07:57:19,326 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7fb8763a5d32,35737,1732521438242 2024-11-25T07:57:19,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T07:57:19,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T07:57:19,347 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7fb8763a5d32,35737,1732521438242 2024-11-25T07:57:19,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:57:19,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T07:57:19,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:57:19,383 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T07:57:19,384 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7fb8763a5d32,35737,1732521438242 from backup master directory 2024-11-25T07:57:19,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7fb8763a5d32,35737,1732521438242 2024-11-25T07:57:19,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T07:57:19,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T07:57:19,397 WARN [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T07:57:19,398 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7fb8763a5d32,35737,1732521438242 2024-11-25T07:57:19,400 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T07:57:19,402 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T07:57:19,470 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/hbase.id] with ID: c3fcd9cc-262d-4c87-a78a-efac10e22ec4 2024-11-25T07:57:19,471 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/.tmp/hbase.id 2024-11-25T07:57:19,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741826_1002 (size=42) 2024-11-25T07:57:19,884 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/.tmp/hbase.id]:[hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/hbase.id] 2024-11-25T07:57:19,939 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T07:57:19,945 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T07:57:19,965 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-25T07:57:20,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:57:20,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:57:20,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741827_1003 (size=196) 2024-11-25T07:57:20,455 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T07:57:20,458 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T07:57:20,474 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:57:20,479 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:57:20,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741828_1004 (size=1189) 2024-11-25T07:57:20,938 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store 2024-11-25T07:57:20,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741829_1005 (size=34) 2024-11-25T07:57:21,368 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T07:57:21,373 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:57:21,374 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T07:57:21,374 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:57:21,374 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:57:21,375 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T07:57:21,376 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:57:21,376 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:57:21,377 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732521441374Disabling compacts and flushes for region at 1732521441374Disabling writes for close at 1732521441375 (+1 ms)Writing region close event to WAL at 1732521441376 (+1 ms)Closed at 1732521441376 2024-11-25T07:57:21,379 WARN [master/7fb8763a5d32:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/.initializing 2024-11-25T07:57:21,379 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/WALs/7fb8763a5d32,35737,1732521438242 2024-11-25T07:57:21,386 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:57:21,400 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7fb8763a5d32%2C35737%2C1732521438242, suffix=, logDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/WALs/7fb8763a5d32,35737,1732521438242, archiveDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/oldWALs, maxLogs=10 2024-11-25T07:57:21,422 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/WALs/7fb8763a5d32,35737,1732521438242/7fb8763a5d32%2C35737%2C1732521438242.1732521441404, exclude list is [], retry=0 2024-11-25T07:57:21,437 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32909,DS-9e5bcbd9-660b-4360-9579-f91c2a9d7df5,DISK] 2024-11-25T07:57:21,439 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T07:57:21,470 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/WALs/7fb8763a5d32,35737,1732521438242/7fb8763a5d32%2C35737%2C1732521438242.1732521441404 2024-11-25T07:57:21,471 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34577:34577)] 2024-11-25T07:57:21,471 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:57:21,472 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:57:21,475 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:57:21,476 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:57:21,509 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:57:21,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T07:57:21,533 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:21,536 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:57:21,536 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:57:21,539 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T07:57:21,539 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:21,540 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:57:21,541 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:57:21,543 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T07:57:21,544 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:21,545 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:57:21,545 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:57:21,548 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T07:57:21,548 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:21,549 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:57:21,549 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:57:21,553 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:57:21,554 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:57:21,559 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:57:21,560 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:57:21,563 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T07:57:21,567 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T07:57:21,571 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:57:21,573 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60621355, jitterRate=-0.09667141735553741}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T07:57:21,582 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732521441487Initializing all the Stores at 1732521441489 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732521441489Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521441490 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521441490Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521441490Cleaning up temporary data from old regions at 1732521441560 (+70 ms)Region opened successfully at 1732521441582 (+22 ms) 2024-11-25T07:57:21,584 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T07:57:21,616 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11975525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7fb8763a5d32/172.17.0.2:0 2024-11-25T07:57:21,643 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T07:57:21,653 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T07:57:21,653 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T07:57:21,655 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T07:57:21,656 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-25T07:57:21,660 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-25T07:57:21,660 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T07:57:21,683 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T07:57:21,690 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T07:57:21,736 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T07:57:21,739 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T07:57:21,742 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T07:57:21,754 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T07:57:21,757 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T07:57:21,761 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T07:57:21,817 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T07:57:21,822 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T07:57:21,830 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T07:57:21,850 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T07:57:21,862 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T07:57:21,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T07:57:21,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T07:57:21,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:57:21,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:57:21,877 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7fb8763a5d32,35737,1732521438242, sessionid=0x10170ff24600000, setting cluster-up flag (Was=false) 2024-11-25T07:57:21,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:57:21,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:57:21,947 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T07:57:21,949 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7fb8763a5d32,35737,1732521438242 2024-11-25T07:57:21,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:57:21,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:57:22,010 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T07:57:22,012 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7fb8763a5d32,35737,1732521438242 2024-11-25T07:57:22,020 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T07:57:22,101 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T07:57:22,111 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T07:57:22,115 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(746): ClusterId : c3fcd9cc-262d-4c87-a78a-efac10e22ec4 2024-11-25T07:57:22,118 DEBUG [RS:0;7fb8763a5d32:41809 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T07:57:22,119 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T07:57:22,124 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7fb8763a5d32,35737,1732521438242 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T07:57:22,148 DEBUG [RS:0;7fb8763a5d32:41809 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T07:57:22,149 DEBUG [RS:0;7fb8763a5d32:41809 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T07:57:22,149 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7fb8763a5d32:0, corePoolSize=5, maxPoolSize=5 2024-11-25T07:57:22,149 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7fb8763a5d32:0, corePoolSize=5, maxPoolSize=5 2024-11-25T07:57:22,150 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7fb8763a5d32:0, corePoolSize=5, maxPoolSize=5 2024-11-25T07:57:22,150 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7fb8763a5d32:0, corePoolSize=5, maxPoolSize=5 2024-11-25T07:57:22,150 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7fb8763a5d32:0, corePoolSize=10, maxPoolSize=10 2024-11-25T07:57:22,150 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7fb8763a5d32:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:57:22,151 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7fb8763a5d32:0, corePoolSize=2, maxPoolSize=2 2024-11-25T07:57:22,151 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7fb8763a5d32:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:57:22,152 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732521472152 2024-11-25T07:57:22,155 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T07:57:22,156 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T07:57:22,157 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T07:57:22,157 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T07:57:22,159 DEBUG [RS:0;7fb8763a5d32:41809 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T07:57:22,159 DEBUG [RS:0;7fb8763a5d32:41809 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b62069, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7fb8763a5d32/172.17.0.2:0 2024-11-25T07:57:22,161 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T07:57:22,161 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T07:57:22,162 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T07:57:22,162 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T07:57:22,164 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:22,165 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T07:57:22,164 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:22,168 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T07:57:22,170 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T07:57:22,171 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T07:57:22,180 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7fb8763a5d32:41809 2024-11-25T07:57:22,183 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T07:57:22,183 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T07:57:22,184 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T07:57:22,186 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(2659): reportForDuty to master=7fb8763a5d32,35737,1732521438242 with port=41809, startcode=1732521439188 2024-11-25T07:57:22,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741831_1007 (size=1321) 2024-11-25T07:57:22,199 DEBUG [RS:0;7fb8763a5d32:41809 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T07:57:22,204 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T07:57:22,205 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T07:57:22,207 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7fb8763a5d32:0:becomeActiveMaster-HFileCleaner.large.0-1732521442206,5,FailOnTimeoutGroup] 2024-11-25T07:57:22,208 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7fb8763a5d32:0:becomeActiveMaster-HFileCleaner.small.0-1732521442207,5,FailOnTimeoutGroup] 2024-11-25T07:57:22,208 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:22,208 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T07:57:22,210 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:22,210 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:22,238 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37869, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T07:57:22,246 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35737 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7fb8763a5d32,41809,1732521439188 2024-11-25T07:57:22,248 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35737 {}] master.ServerManager(517): Registering regionserver=7fb8763a5d32,41809,1732521439188 2024-11-25T07:57:22,264 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574 2024-11-25T07:57:22,264 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37877 2024-11-25T07:57:22,264 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T07:57:22,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T07:57:22,274 DEBUG [RS:0;7fb8763a5d32:41809 {}] zookeeper.ZKUtil(111): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7fb8763a5d32,41809,1732521439188 2024-11-25T07:57:22,274 WARN [RS:0;7fb8763a5d32:41809 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T07:57:22,274 INFO [RS:0;7fb8763a5d32:41809 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:57:22,275 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/WALs/7fb8763a5d32,41809,1732521439188 2024-11-25T07:57:22,276 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7fb8763a5d32,41809,1732521439188] 2024-11-25T07:57:22,306 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T07:57:22,326 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T07:57:22,331 INFO [RS:0;7fb8763a5d32:41809 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T07:57:22,331 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:22,333 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T07:57:22,339 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T07:57:22,341 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:22,341 DEBUG [RS:0;7fb8763a5d32:41809 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7fb8763a5d32:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:57:22,341 DEBUG [RS:0;7fb8763a5d32:41809 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7fb8763a5d32:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:57:22,342 DEBUG [RS:0;7fb8763a5d32:41809 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:57:22,342 DEBUG [RS:0;7fb8763a5d32:41809 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7fb8763a5d32:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:57:22,342 DEBUG [RS:0;7fb8763a5d32:41809 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7fb8763a5d32:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:57:22,342 DEBUG [RS:0;7fb8763a5d32:41809 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7fb8763a5d32:0, corePoolSize=2, maxPoolSize=2 2024-11-25T07:57:22,342 DEBUG [RS:0;7fb8763a5d32:41809 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7fb8763a5d32:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:57:22,342 DEBUG [RS:0;7fb8763a5d32:41809 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7fb8763a5d32:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:57:22,343 DEBUG [RS:0;7fb8763a5d32:41809 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7fb8763a5d32:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:57:22,343 DEBUG [RS:0;7fb8763a5d32:41809 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7fb8763a5d32:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:57:22,343 DEBUG [RS:0;7fb8763a5d32:41809 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7fb8763a5d32:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:57:22,343 DEBUG [RS:0;7fb8763a5d32:41809 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7fb8763a5d32:0, corePoolSize=1, maxPoolSize=1 2024-11-25T07:57:22,343 DEBUG [RS:0;7fb8763a5d32:41809 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7fb8763a5d32:0, corePoolSize=3, maxPoolSize=3 2024-11-25T07:57:22,344 DEBUG [RS:0;7fb8763a5d32:41809 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7fb8763a5d32:0, corePoolSize=3, maxPoolSize=3 2024-11-25T07:57:22,346 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:22,347 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:22,347 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:22,347 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:22,347 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:22,347 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.ChoreService(168): Chore ScheduledChore name=7fb8763a5d32,41809,1732521439188-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T07:57:22,369 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T07:57:22,371 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.ChoreService(168): Chore ScheduledChore name=7fb8763a5d32,41809,1732521439188-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:22,372 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:22,372 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.Replication(171): 7fb8763a5d32,41809,1732521439188 started 2024-11-25T07:57:22,392 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:22,392 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1482): Serving as 7fb8763a5d32,41809,1732521439188, RpcServer on 7fb8763a5d32/172.17.0.2:41809, sessionid=0x10170ff24600001 2024-11-25T07:57:22,393 DEBUG [RS:0;7fb8763a5d32:41809 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T07:57:22,393 DEBUG [RS:0;7fb8763a5d32:41809 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7fb8763a5d32,41809,1732521439188 2024-11-25T07:57:22,393 DEBUG [RS:0;7fb8763a5d32:41809 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7fb8763a5d32,41809,1732521439188' 2024-11-25T07:57:22,393 DEBUG [RS:0;7fb8763a5d32:41809 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T07:57:22,395 DEBUG [RS:0;7fb8763a5d32:41809 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T07:57:22,395 DEBUG [RS:0;7fb8763a5d32:41809 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T07:57:22,396 DEBUG [RS:0;7fb8763a5d32:41809 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T07:57:22,396 DEBUG [RS:0;7fb8763a5d32:41809 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7fb8763a5d32,41809,1732521439188 2024-11-25T07:57:22,396 DEBUG [RS:0;7fb8763a5d32:41809 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7fb8763a5d32,41809,1732521439188' 2024-11-25T07:57:22,396 DEBUG [RS:0;7fb8763a5d32:41809 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T07:57:22,397 DEBUG [RS:0;7fb8763a5d32:41809 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T07:57:22,397 DEBUG [RS:0;7fb8763a5d32:41809 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T07:57:22,397 INFO [RS:0;7fb8763a5d32:41809 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T07:57:22,398 INFO [RS:0;7fb8763a5d32:41809 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T07:57:22,594 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T07:57:22,595 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574 2024-11-25T07:57:22,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741832_1008 (size=32) 2024-11-25T07:57:23,011 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:57:23,014 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T07:57:23,017 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T07:57:23,018 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:23,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:57:23,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T07:57:23,022 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T07:57:23,023 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:23,024 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:57:23,024 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T07:57:23,027 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T07:57:23,027 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:23,028 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:57:23,028 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T07:57:23,031 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T07:57:23,031 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:23,032 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:57:23,032 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T07:57:23,034 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740 2024-11-25T07:57:23,035 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740 2024-11-25T07:57:23,038 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T07:57:23,038 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T07:57:23,039 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T07:57:23,042 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T07:57:23,046 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:57:23,047 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61617952, jitterRate=-0.08182096481323242}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T07:57:23,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732521443011Initializing all the Stores at 1732521443013 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732521443013Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732521443014 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521443014Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732521443014Cleaning up temporary data from old regions at 1732521443038 (+24 ms)Region opened successfully at 1732521443050 (+12 ms) 2024-11-25T07:57:23,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T07:57:23,051 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T07:57:23,051 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T07:57:23,051 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T07:57:23,051 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T07:57:23,052 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T07:57:23,053 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732521443050Disabling compacts and flushes for region at 1732521443050Disabling writes for close at 1732521443051 (+1 ms)Writing region close event to WAL at 1732521443052 (+1 ms)Closed at 1732521443052 2024-11-25T07:57:23,056 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T07:57:23,056 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T07:57:23,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T07:57:23,071 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T07:57:23,074 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T07:57:23,225 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:24,226 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:25,227 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:26,228 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:27,229 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:28,230 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:28,649 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T07:57:28,650 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T07:57:29,230 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:30,231 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:31,232 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:32,232 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:33,233 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:34,234 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:35,235 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:36,235 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:37,236 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:38,237 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:39,237 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:40,238 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:41,239 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:42,239 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:43,240 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:44,241 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:45,242 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:46,242 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:47,243 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:47,305 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T07:57:48,244 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:49,245 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:50,245 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:51,246 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:52,247 WARN [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T07:57:52,404 INFO [RS:0;7fb8763a5d32:41809 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T07:57:52,409 INFO [RS:0;7fb8763a5d32:41809 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7fb8763a5d32%2C41809%2C1732521439188, suffix=, logDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/WALs/7fb8763a5d32,41809,1732521439188, archiveDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/oldWALs, maxLogs=32 2024-11-25T07:57:52,426 DEBUG [RS:0;7fb8763a5d32:41809 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/WALs/7fb8763a5d32,41809,1732521439188/7fb8763a5d32%2C41809%2C1732521439188.1732521472412, exclude list is [], retry=0 2024-11-25T07:57:52,430 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32909,DS-9e5bcbd9-660b-4360-9579-f91c2a9d7df5,DISK] 2024-11-25T07:57:52,437 INFO [RS:0;7fb8763a5d32:41809 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/WALs/7fb8763a5d32,41809,1732521439188/7fb8763a5d32%2C41809%2C1732521439188.1732521472412 2024-11-25T07:57:52,438 DEBUG [RS:0;7fb8763a5d32:41809 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34577:34577)] 2024-11-25T07:57:52,499 DEBUG [7fb8763a5d32:35737 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-25T07:57:52,513 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:57:52,523 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7fb8763a5d32,41809,1732521439188, state=OPENING 2024-11-25T07:57:52,576 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T07:57:52,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:57:52,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:57:52,588 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T07:57:52,588 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T07:57:52,590 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T07:57:52,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7fb8763a5d32,41809,1732521439188}] 2024-11-25T07:57:52,776 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T07:57:52,779 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55973, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T07:57:52,791 INFO [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T07:57:52,791 INFO [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T07:57:52,792 INFO [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T07:57:52,796 INFO [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7fb8763a5d32%2C41809%2C1732521439188.meta, suffix=.meta, logDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/WALs/7fb8763a5d32,41809,1732521439188, archiveDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/oldWALs, maxLogs=32 2024-11-25T07:57:52,811 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/WALs/7fb8763a5d32,41809,1732521439188/7fb8763a5d32%2C41809%2C1732521439188.meta.1732521472798.meta, exclude list is [], retry=0 2024-11-25T07:57:52,815 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32909,DS-9e5bcbd9-660b-4360-9579-f91c2a9d7df5,DISK] 2024-11-25T07:57:52,826 INFO [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/WALs/7fb8763a5d32,41809,1732521439188/7fb8763a5d32%2C41809%2C1732521439188.meta.1732521472798.meta 2024-11-25T07:57:52,826 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34577:34577)] 2024-11-25T07:57:52,827 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:57:52,829 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T07:57:52,832 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T07:57:52,836 INFO [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T07:57:52,841 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T07:57:52,841 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:57:52,841 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T07:57:52,842 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T07:57:52,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T07:57:52,847 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T07:57:52,847 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:52,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:57:52,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T07:57:52,851 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T07:57:52,851 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:52,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:57:52,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T07:57:52,854 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T07:57:52,854 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:52,855 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:57:52,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T07:57:52,857 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T07:57:52,858 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:52,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T07:57:52,859 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T07:57:52,861 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740 2024-11-25T07:57:52,864 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740 2024-11-25T07:57:52,867 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T07:57:52,867 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T07:57:52,868 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T07:57:52,871 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T07:57:52,873 INFO [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71553582, jitterRate=0.06623145937919617}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T07:57:52,874 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T07:57:52,875 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732521472842Writing region info on filesystem at 1732521472842Initializing all the Stores at 1732521472844 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732521472844Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732521472845 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521472845Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732521472845Cleaning up temporary data from old regions at 1732521472868 (+23 ms)Running coprocessor post-open hooks at 1732521472874 (+6 ms)Region opened successfully at 1732521472875 (+1 ms) 2024-11-25T07:57:52,883 INFO [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732521472766 2024-11-25T07:57:52,896 DEBUG [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T07:57:52,897 INFO [RS_OPEN_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T07:57:52,899 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:57:52,902 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7fb8763a5d32,41809,1732521439188, state=OPEN 2024-11-25T07:57:53,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T07:57:53,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T07:57:53,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T07:57:53,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T07:57:53,046 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7fb8763a5d32,41809,1732521439188 2024-11-25T07:57:53,054 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T07:57:53,054 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7fb8763a5d32,41809,1732521439188 in 454 msec 2024-11-25T07:57:53,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T07:57:53,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.9940 sec 2024-11-25T07:57:53,064 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T07:57:53,064 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T07:57:53,086 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T07:57:53,088 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7fb8763a5d32,41809,1732521439188, seqNum=-1] 2024-11-25T07:57:53,109 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T07:57:53,112 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44405, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T07:57:53,133 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 31.0760 sec 2024-11-25T07:57:53,133 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732521473133, completionTime=-1 2024-11-25T07:57:53,137 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-25T07:57:53,137 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T07:57:53,168 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-25T07:57:53,168 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732521533168 2024-11-25T07:57:53,168 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732521593168 2024-11-25T07:57:53,168 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 31 msec 2024-11-25T07:57:53,170 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7fb8763a5d32,35737,1732521438242-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:53,171 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7fb8763a5d32,35737,1732521438242-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:53,171 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7fb8763a5d32,35737,1732521438242-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:53,172 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7fb8763a5d32:35737, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:53,173 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:53,173 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:53,179 DEBUG [master/7fb8763a5d32:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T07:57:53,205 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.806sec 2024-11-25T07:57:53,206 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T07:57:53,207 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T07:57:53,209 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T07:57:53,209 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T07:57:53,210 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T07:57:53,211 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7fb8763a5d32,35737,1732521438242-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T07:57:53,211 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7fb8763a5d32,35737,1732521438242-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T07:57:53,221 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T07:57:53,222 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T07:57:53,222 INFO [master/7fb8763a5d32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7fb8763a5d32,35737,1732521438242-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T07:57:53,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573d5973, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T07:57:53,258 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T07:57:53,259 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T07:57:53,262 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7fb8763a5d32,35737,-1 for getting cluster id 2024-11-25T07:57:53,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T07:57:53,273 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c3fcd9cc-262d-4c87-a78a-efac10e22ec4' 2024-11-25T07:57:53,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T07:57:53,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c3fcd9cc-262d-4c87-a78a-efac10e22ec4" 2024-11-25T07:57:53,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b0ddee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T07:57:53,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7fb8763a5d32,35737,-1] 2024-11-25T07:57:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T07:57:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:57:53,285 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T07:57:53,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1928876b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T07:57:53,290 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T07:57:53,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7fb8763a5d32,41809,1732521439188, seqNum=-1] 2024-11-25T07:57:53,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T07:57:53,302 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T07:57:53,323 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7fb8763a5d32,35737,1732521438242 2024-11-25T07:57:53,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2246aef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T07:57:53,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7fb8763a5d32,35737,-1 for getting cluster id 2024-11-25T07:57:53,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T07:57:53,328 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c3fcd9cc-262d-4c87-a78a-efac10e22ec4' 2024-11-25T07:57:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T07:57:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c3fcd9cc-262d-4c87-a78a-efac10e22ec4" 2024-11-25T07:57:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58c4d5a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T07:57:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7fb8763a5d32,35737,-1] 2024-11-25T07:57:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T07:57:53,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:57:53,333 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T07:57:53,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@666749f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T07:57:53,350 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=221, ProcessCount=11, AvailableMemoryMB=4389 2024-11-25T07:57:53,357 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T07:57:53,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7fb8763a5d32,35737,1732521438242 2024-11-25T07:57:53,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5e14daea 2024-11-25T07:57:53,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T07:57:53,370 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T07:57:53,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T07:57:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-25T07:57:53,405 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T07:57:53,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-25T07:57:53,409 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:53,412 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T07:57:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T07:57:53,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741835_1011 (size=428) 2024-11-25T07:57:53,851 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e450be747e11bc58612b8b5baa075bfa, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574 2024-11-25T07:57:53,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741836_1012 (size=79) 2024-11-25T07:57:54,265 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:57:54,265 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing e450be747e11bc58612b8b5baa075bfa, disabling compactions & flushes 2024-11-25T07:57:54,266 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. 2024-11-25T07:57:54,266 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. 2024-11-25T07:57:54,266 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. after waiting 0 ms 2024-11-25T07:57:54,266 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. 2024-11-25T07:57:54,266 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. 2024-11-25T07:57:54,266 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for e450be747e11bc58612b8b5baa075bfa: Waiting for close lock at 1732521474265Disabling compacts and flushes for region at 1732521474265Disabling writes for close at 1732521474266 (+1 ms)Writing region close event to WAL at 1732521474266Closed at 1732521474266 2024-11-25T07:57:54,270 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T07:57:54,277 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732521474270"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732521474270"}]},"ts":"1732521474270"} 2024-11-25T07:57:54,282 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T07:57:54,285 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T07:57:54,288 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521474285"}]},"ts":"1732521474285"} 2024-11-25T07:57:54,293 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-25T07:57:54,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e450be747e11bc58612b8b5baa075bfa, ASSIGN}] 2024-11-25T07:57:54,298 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e450be747e11bc58612b8b5baa075bfa, ASSIGN 2024-11-25T07:57:54,300 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e450be747e11bc58612b8b5baa075bfa, ASSIGN; state=OFFLINE, location=7fb8763a5d32,41809,1732521439188; forceNewPlan=false, retain=false 2024-11-25T07:57:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T07:57:54,452 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e450be747e11bc58612b8b5baa075bfa, regionState=OPENING, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:57:54,457 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e450be747e11bc58612b8b5baa075bfa, ASSIGN because future has completed 2024-11-25T07:57:54,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e450be747e11bc58612b8b5baa075bfa, server=7fb8763a5d32,41809,1732521439188}] 2024-11-25T07:57:54,621 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. 2024-11-25T07:57:54,622 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e450be747e11bc58612b8b5baa075bfa, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:57:54,622 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync e450be747e11bc58612b8b5baa075bfa 2024-11-25T07:57:54,622 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:57:54,623 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e450be747e11bc58612b8b5baa075bfa 2024-11-25T07:57:54,623 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e450be747e11bc58612b8b5baa075bfa 2024-11-25T07:57:54,626 INFO [StoreOpener-e450be747e11bc58612b8b5baa075bfa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e450be747e11bc58612b8b5baa075bfa 2024-11-25T07:57:54,629 INFO [StoreOpener-e450be747e11bc58612b8b5baa075bfa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e450be747e11bc58612b8b5baa075bfa columnFamilyName testFamily 2024-11-25T07:57:54,629 DEBUG [StoreOpener-e450be747e11bc58612b8b5baa075bfa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:54,630 INFO [StoreOpener-e450be747e11bc58612b8b5baa075bfa-1 {}] regionserver.HStore(327): Store=e450be747e11bc58612b8b5baa075bfa/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:57:54,631 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e450be747e11bc58612b8b5baa075bfa 2024-11-25T07:57:54,632 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e450be747e11bc58612b8b5baa075bfa 2024-11-25T07:57:54,633 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e450be747e11bc58612b8b5baa075bfa 2024-11-25T07:57:54,634 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e450be747e11bc58612b8b5baa075bfa 2024-11-25T07:57:54,634 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e450be747e11bc58612b8b5baa075bfa 2024-11-25T07:57:54,637 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e450be747e11bc58612b8b5baa075bfa 2024-11-25T07:57:54,641 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e450be747e11bc58612b8b5baa075bfa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:57:54,642 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e450be747e11bc58612b8b5baa075bfa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62715507, jitterRate=-0.06546612083911896}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:57:54,642 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e450be747e11bc58612b8b5baa075bfa 2024-11-25T07:57:54,643 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e450be747e11bc58612b8b5baa075bfa: Running coprocessor pre-open hook at 1732521474623Writing region info on filesystem at 1732521474623Initializing all the Stores at 1732521474625 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521474625Cleaning up temporary data from old regions at 1732521474634 (+9 ms)Running coprocessor post-open hooks at 1732521474642 (+8 ms)Region opened successfully at 1732521474643 (+1 ms) 2024-11-25T07:57:54,646 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa., pid=6, masterSystemTime=1732521474614 2024-11-25T07:57:54,650 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. 2024-11-25T07:57:54,651 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. 2024-11-25T07:57:54,652 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e450be747e11bc58612b8b5baa075bfa, regionState=OPEN, openSeqNum=2, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:57:54,657 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e450be747e11bc58612b8b5baa075bfa, server=7fb8763a5d32,41809,1732521439188 because future has completed 2024-11-25T07:57:54,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T07:57:54,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e450be747e11bc58612b8b5baa075bfa, server=7fb8763a5d32,41809,1732521439188 in 201 msec 2024-11-25T07:57:54,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T07:57:54,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e450be747e11bc58612b8b5baa075bfa, ASSIGN in 369 msec 2024-11-25T07:57:54,671 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T07:57:54,671 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521474671"}]},"ts":"1732521474671"} 2024-11-25T07:57:54,675 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-25T07:57:54,677 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T07:57:54,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2810 sec 2024-11-25T07:57:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T07:57:56,456 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-25T07:57:56,456 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-25T07:57:56,458 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:57:56,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-25T07:57:56,467 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:57:56,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-25T07:57:56,472 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T07:57:56,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7fb8763a5d32,41809,1732521439188, seqNum=-1] 2024-11-25T07:57:56,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T07:57:56,481 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T07:57:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2] 2024-11-25T07:57:56,504 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T07:57:56,508 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-25T07:57:56,514 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-25T07:57:56,518 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-25T07:57:56,521 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-25T07:57:56,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-601213537888501752 2024-11-25T07:57:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46980 deadline: 1732521479520, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T07:57:56,549 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-25T07:57:56,551 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-25T07:57:56,552 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T07:57:56,554 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T07:57:56,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-601213537888501750 2024-11-25T07:57:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46980 deadline: 1732521479553, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T07:57:56,559 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T07:57:56,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-601213537888501749 2024-11-25T07:57:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46980 deadline: 1732521479559, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T07:57:56,566 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T07:57:56,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-601213537888501748 2024-11-25T07:57:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46980 deadline: 1732521479565, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T07:57:56,571 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T07:57:56,582 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-727370367_22 at /127.0.0.1:35100 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=228 (was 221) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4346 (was 4389) 2024-11-25T07:57:56,591 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=228, ProcessCount=11, AvailableMemoryMB=4346 2024-11-25T07:57:56,595 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35737 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T07:57:56,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35737 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-25T07:57:56,600 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T07:57:56,601 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:56,601 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35737 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-25T07:57:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T07:57:56,603 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T07:57:56,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741837_1013 (size=414) 2024-11-25T07:57:57,017 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ffa3712a288e16bfac8069191f2eda48, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574 2024-11-25T07:57:57,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741838_1014 (size=65) 2024-11-25T07:57:57,440 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:57:57,440 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing ffa3712a288e16bfac8069191f2eda48, disabling compactions & flushes 2024-11-25T07:57:57,440 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. 2024-11-25T07:57:57,440 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. 2024-11-25T07:57:57,441 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. after waiting 0 ms 2024-11-25T07:57:57,441 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. 2024-11-25T07:57:57,441 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. 2024-11-25T07:57:57,441 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for ffa3712a288e16bfac8069191f2eda48: Waiting for close lock at 1732521477440Disabling compacts and flushes for region at 1732521477440Disabling writes for close at 1732521477441 (+1 ms)Writing region close event to WAL at 1732521477441Closed at 1732521477441 2024-11-25T07:57:57,443 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T07:57:57,444 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732521477443"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732521477443"}]},"ts":"1732521477443"} 2024-11-25T07:57:57,448 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T07:57:57,450 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T07:57:57,450 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521477450"}]},"ts":"1732521477450"} 2024-11-25T07:57:57,453 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-25T07:57:57,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ffa3712a288e16bfac8069191f2eda48, ASSIGN}] 2024-11-25T07:57:57,456 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ffa3712a288e16bfac8069191f2eda48, ASSIGN 2024-11-25T07:57:57,457 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ffa3712a288e16bfac8069191f2eda48, ASSIGN; state=OFFLINE, location=7fb8763a5d32,41809,1732521439188; forceNewPlan=false, retain=false 2024-11-25T07:57:57,608 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ffa3712a288e16bfac8069191f2eda48, regionState=OPENING, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:57:57,612 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ffa3712a288e16bfac8069191f2eda48, ASSIGN because future has completed 2024-11-25T07:57:57,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffa3712a288e16bfac8069191f2eda48, server=7fb8763a5d32,41809,1732521439188}] 2024-11-25T07:57:57,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T07:57:57,773 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. 2024-11-25T07:57:57,774 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ffa3712a288e16bfac8069191f2eda48, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:57:57,774 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:57:57,774 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:57:57,775 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:57:57,775 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:57:57,777 INFO [StoreOpener-ffa3712a288e16bfac8069191f2eda48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:57:57,780 INFO [StoreOpener-ffa3712a288e16bfac8069191f2eda48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ffa3712a288e16bfac8069191f2eda48 columnFamilyName testFamily 2024-11-25T07:57:57,780 DEBUG [StoreOpener-ffa3712a288e16bfac8069191f2eda48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:57:57,780 INFO [StoreOpener-ffa3712a288e16bfac8069191f2eda48-1 {}] regionserver.HStore(327): Store=ffa3712a288e16bfac8069191f2eda48/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:57:57,781 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:57:57,782 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScanner/ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:57:57,783 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScanner/ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:57:57,783 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:57:57,783 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:57:57,787 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:57:57,790 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScanner/ffa3712a288e16bfac8069191f2eda48/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:57:57,791 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened ffa3712a288e16bfac8069191f2eda48; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67751325, jitterRate=0.009573414921760559}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:57:57,791 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:57:57,793 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ffa3712a288e16bfac8069191f2eda48: Running coprocessor pre-open hook at 1732521477775Writing region info on filesystem at 1732521477775Initializing all the Stores at 1732521477777 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521477777Cleaning up temporary data from old regions at 1732521477784 (+7 ms)Running coprocessor post-open hooks at 1732521477791 (+7 ms)Region opened successfully at 1732521477792 (+1 ms) 2024-11-25T07:57:57,794 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48., pid=9, masterSystemTime=1732521477767 2024-11-25T07:57:57,798 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. 2024-11-25T07:57:57,798 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. 2024-11-25T07:57:57,799 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ffa3712a288e16bfac8069191f2eda48, regionState=OPEN, openSeqNum=2, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:57:57,802 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ffa3712a288e16bfac8069191f2eda48, server=7fb8763a5d32,41809,1732521439188 because future has completed 2024-11-25T07:57:57,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-25T07:57:57,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure ffa3712a288e16bfac8069191f2eda48, server=7fb8763a5d32,41809,1732521439188 in 191 msec 2024-11-25T07:57:57,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T07:57:57,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ffa3712a288e16bfac8069191f2eda48, ASSIGN in 355 msec 2024-11-25T07:57:57,815 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T07:57:57,815 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521477815"}]},"ts":"1732521477815"} 2024-11-25T07:57:57,818 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-25T07:57:57,820 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T07:57:57,823 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2240 sec 2024-11-25T07:57:58,381 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T07:57:58,420 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-25T07:57:58,422 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T07:57:58,423 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-25T07:57:58,649 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T07:57:58,649 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T07:57:58,651 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T07:57:58,651 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T07:57:58,652 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-25T07:57:58,652 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T07:57:58,653 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-25T07:57:58,653 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-25T07:57:59,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T07:57:59,645 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-25T07:57:59,645 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-25T07:57:59,645 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:57:59,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-25T07:57:59,651 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:57:59,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-25T07:57:59,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2] 2024-11-25T07:57:59,670 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T07:57:59,671 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-25T07:57:59,675 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-25T07:57:59,677 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-25T07:58:03,277 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T07:58:05,695 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=7fb8763a5d32:41809 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6015ms, rpcTimeout=5999ms 2024-11-25T07:58:05,696 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6015ms, rpcTimeout=5999ms 2024-11-25T07:58:05,696 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 from cache 2024-11-25T07:58:05,705 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=216 (was 228), ProcessCount=11 (was 11), AvailableMemoryMB=4338 (was 4346) 2024-11-25T07:58:05,714 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=216, ProcessCount=11, AvailableMemoryMB=4338 2024-11-25T07:58:05,717 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35737 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T07:58:05,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35737 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-25T07:58:05,720 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T07:58:05,720 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:05,721 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35737 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-25T07:58:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-25T07:58:05,722 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T07:58:05,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741839_1015 (size=412) 2024-11-25T07:58:06,135 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 470929e933e6a8a588c3fc1fd5ba5065, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574 2024-11-25T07:58:06,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741840_1016 (size=63) 2024-11-25T07:58:06,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-25T07:58:06,546 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:06,546 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 470929e933e6a8a588c3fc1fd5ba5065, disabling compactions & flushes 2024-11-25T07:58:06,546 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. 2024-11-25T07:58:06,547 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. 2024-11-25T07:58:06,547 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. after waiting 0 ms 2024-11-25T07:58:06,547 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. 2024-11-25T07:58:06,547 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. 2024-11-25T07:58:06,547 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 470929e933e6a8a588c3fc1fd5ba5065: Waiting for close lock at 1732521486546Disabling compacts and flushes for region at 1732521486546Disabling writes for close at 1732521486547 (+1 ms)Writing region close event to WAL at 1732521486547Closed at 1732521486547 2024-11-25T07:58:06,549 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T07:58:06,550 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732521486550"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732521486550"}]},"ts":"1732521486550"} 2024-11-25T07:58:06,554 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T07:58:06,556 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T07:58:06,556 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521486556"}]},"ts":"1732521486556"} 2024-11-25T07:58:06,559 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-25T07:58:06,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=470929e933e6a8a588c3fc1fd5ba5065, ASSIGN}] 2024-11-25T07:58:06,562 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=470929e933e6a8a588c3fc1fd5ba5065, ASSIGN 2024-11-25T07:58:06,563 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=470929e933e6a8a588c3fc1fd5ba5065, ASSIGN; state=OFFLINE, location=7fb8763a5d32,41809,1732521439188; forceNewPlan=false, retain=false 2024-11-25T07:58:06,714 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=470929e933e6a8a588c3fc1fd5ba5065, regionState=OPENING, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:06,719 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=470929e933e6a8a588c3fc1fd5ba5065, ASSIGN because future has completed 2024-11-25T07:58:06,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 470929e933e6a8a588c3fc1fd5ba5065, server=7fb8763a5d32,41809,1732521439188}] 2024-11-25T07:58:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-25T07:58:06,880 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. 2024-11-25T07:58:06,881 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 470929e933e6a8a588c3fc1fd5ba5065, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:58:06,881 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 470929e933e6a8a588c3fc1fd5ba5065 2024-11-25T07:58:06,881 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:06,881 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 470929e933e6a8a588c3fc1fd5ba5065 2024-11-25T07:58:06,882 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 470929e933e6a8a588c3fc1fd5ba5065 2024-11-25T07:58:06,884 INFO [StoreOpener-470929e933e6a8a588c3fc1fd5ba5065-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 470929e933e6a8a588c3fc1fd5ba5065 2024-11-25T07:58:06,887 INFO [StoreOpener-470929e933e6a8a588c3fc1fd5ba5065-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 470929e933e6a8a588c3fc1fd5ba5065 columnFamilyName testFamily 2024-11-25T07:58:06,887 DEBUG [StoreOpener-470929e933e6a8a588c3fc1fd5ba5065-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:06,888 INFO [StoreOpener-470929e933e6a8a588c3fc1fd5ba5065-1 {}] regionserver.HStore(327): Store=470929e933e6a8a588c3fc1fd5ba5065/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:58:06,888 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 470929e933e6a8a588c3fc1fd5ba5065 2024-11-25T07:58:06,890 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNextAsync/470929e933e6a8a588c3fc1fd5ba5065 2024-11-25T07:58:06,891 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNextAsync/470929e933e6a8a588c3fc1fd5ba5065 2024-11-25T07:58:06,892 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 470929e933e6a8a588c3fc1fd5ba5065 2024-11-25T07:58:06,892 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 470929e933e6a8a588c3fc1fd5ba5065 2024-11-25T07:58:06,895 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 470929e933e6a8a588c3fc1fd5ba5065 2024-11-25T07:58:06,898 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNextAsync/470929e933e6a8a588c3fc1fd5ba5065/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:58:06,899 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 470929e933e6a8a588c3fc1fd5ba5065; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60286449, jitterRate=-0.10166190564632416}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:58:06,899 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 470929e933e6a8a588c3fc1fd5ba5065 2024-11-25T07:58:06,900 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 470929e933e6a8a588c3fc1fd5ba5065: Running coprocessor pre-open hook at 1732521486882Writing region info on filesystem at 1732521486882Initializing all the Stores at 1732521486883 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521486884 (+1 ms)Cleaning up temporary data from old regions at 1732521486892 (+8 ms)Running coprocessor post-open hooks at 1732521486899 (+7 ms)Region opened successfully at 1732521486900 (+1 ms) 2024-11-25T07:58:06,902 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065., pid=12, masterSystemTime=1732521486874 2024-11-25T07:58:06,905 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. 2024-11-25T07:58:06,905 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. 2024-11-25T07:58:06,906 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=470929e933e6a8a588c3fc1fd5ba5065, regionState=OPEN, openSeqNum=2, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:06,910 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 470929e933e6a8a588c3fc1fd5ba5065, server=7fb8763a5d32,41809,1732521439188 because future has completed 2024-11-25T07:58:06,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-25T07:58:06,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 470929e933e6a8a588c3fc1fd5ba5065, server=7fb8763a5d32,41809,1732521439188 in 193 msec 2024-11-25T07:58:06,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-25T07:58:06,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=470929e933e6a8a588c3fc1fd5ba5065, ASSIGN in 356 msec 2024-11-25T07:58:06,921 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T07:58:06,922 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521486921"}]},"ts":"1732521486921"} 2024-11-25T07:58:06,925 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-25T07:58:06,926 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T07:58:06,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2090 sec 2024-11-25T07:58:08,649 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-25T07:58:08,649 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T07:58:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-25T07:58:08,745 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-25T07:58:08,745 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-25T07:58:08,745 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:08,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-25T07:58:08,752 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:08,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-25T07:58:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2] 2024-11-25T07:58:08,770 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T07:58:08,771 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-25T07:58:08,775 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-25T07:58:08,778 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-25T07:58:08,780 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-25T07:58:08,781 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-25T07:58:10,426 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-25T07:58:12,383 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T07:58:14,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-601213537888501739' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-25T07:58:14,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41809 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:46980 deadline: 1732521500785, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-601213537888501739' does not exist 2024-11-25T07:58:14,793 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 7fb8763a5d32,41809,1732521439188 for closing scanner id = -601213537888501739 for 470929e933e6a8a588c3fc1fd5ba5065 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-601213537888501739' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:58:14,795 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=206 (was 216), ProcessCount=11 (was 11), AvailableMemoryMB=4321 (was 4338) 2024-11-25T07:58:14,804 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=206, ProcessCount=11, AvailableMemoryMB=4321 2024-11-25T07:58:14,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T07:58:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-25T07:58:14,810 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T07:58:14,811 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:14,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-25T07:58:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T07:58:14,813 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T07:58:14,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741841_1017 (size=423) 2024-11-25T07:58:15,225 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 148d94d6f40d416655c1aa51d449d387, NAME => 'testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574 2024-11-25T07:58:15,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741842_1018 (size=74) 2024-11-25T07:58:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-25T07:58:15,639 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:15,639 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 148d94d6f40d416655c1aa51d449d387, disabling compactions & flushes 2024-11-25T07:58:15,640 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. 2024-11-25T07:58:15,640 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. 2024-11-25T07:58:15,640 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. after waiting 0 ms 2024-11-25T07:58:15,640 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. 2024-11-25T07:58:15,640 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. 2024-11-25T07:58:15,640 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 148d94d6f40d416655c1aa51d449d387: Waiting for close lock at 1732521495639Disabling compacts and flushes for region at 1732521495639Disabling writes for close at 1732521495640 (+1 ms)Writing region close event to WAL at 1732521495640Closed at 1732521495640 2024-11-25T07:58:15,642 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T07:58:15,643 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732521495642"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732521495642"}]},"ts":"1732521495642"} 2024-11-25T07:58:15,647 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T07:58:15,649 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T07:58:15,649 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521495649"}]},"ts":"1732521495649"} 2024-11-25T07:58:15,653 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-25T07:58:15,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=148d94d6f40d416655c1aa51d449d387, ASSIGN}] 2024-11-25T07:58:15,655 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=148d94d6f40d416655c1aa51d449d387, ASSIGN 2024-11-25T07:58:15,657 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=148d94d6f40d416655c1aa51d449d387, ASSIGN; state=OFFLINE, location=7fb8763a5d32,41809,1732521439188; forceNewPlan=false, retain=false 2024-11-25T07:58:15,808 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=148d94d6f40d416655c1aa51d449d387, regionState=OPENING, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:15,813 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=148d94d6f40d416655c1aa51d449d387, ASSIGN because future has completed 2024-11-25T07:58:15,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 148d94d6f40d416655c1aa51d449d387, server=7fb8763a5d32,41809,1732521439188}] 2024-11-25T07:58:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T07:58:15,974 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. 2024-11-25T07:58:15,974 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 148d94d6f40d416655c1aa51d449d387, NAME => 'testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:58:15,975 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 148d94d6f40d416655c1aa51d449d387 2024-11-25T07:58:15,975 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:15,975 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 148d94d6f40d416655c1aa51d449d387 2024-11-25T07:58:15,975 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 148d94d6f40d416655c1aa51d449d387 2024-11-25T07:58:15,977 INFO [StoreOpener-148d94d6f40d416655c1aa51d449d387-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 148d94d6f40d416655c1aa51d449d387 2024-11-25T07:58:15,981 INFO [StoreOpener-148d94d6f40d416655c1aa51d449d387-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 148d94d6f40d416655c1aa51d449d387 columnFamilyName testFamily 2024-11-25T07:58:15,981 DEBUG [StoreOpener-148d94d6f40d416655c1aa51d449d387-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:15,982 INFO [StoreOpener-148d94d6f40d416655c1aa51d449d387-1 {}] regionserver.HStore(327): Store=148d94d6f40d416655c1aa51d449d387/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:58:15,982 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 148d94d6f40d416655c1aa51d449d387 2024-11-25T07:58:15,984 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextException/148d94d6f40d416655c1aa51d449d387 2024-11-25T07:58:15,985 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextException/148d94d6f40d416655c1aa51d449d387 2024-11-25T07:58:15,985 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 148d94d6f40d416655c1aa51d449d387 2024-11-25T07:58:15,985 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 148d94d6f40d416655c1aa51d449d387 2024-11-25T07:58:15,989 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 148d94d6f40d416655c1aa51d449d387 2024-11-25T07:58:15,997 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextException/148d94d6f40d416655c1aa51d449d387/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:58:15,999 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 148d94d6f40d416655c1aa51d449d387; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69736794, jitterRate=0.03915920853614807}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:58:15,999 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 148d94d6f40d416655c1aa51d449d387 2024-11-25T07:58:15,999 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 148d94d6f40d416655c1aa51d449d387: Running coprocessor pre-open hook at 1732521495975Writing region info on filesystem at 1732521495975Initializing all the Stores at 1732521495977 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521495977Cleaning up temporary data from old regions at 1732521495985 (+8 ms)Running coprocessor post-open hooks at 1732521495999 (+14 ms)Region opened successfully at 1732521495999 2024-11-25T07:58:16,001 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387., pid=15, masterSystemTime=1732521495968 2024-11-25T07:58:16,004 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. 2024-11-25T07:58:16,004 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. 2024-11-25T07:58:16,005 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=148d94d6f40d416655c1aa51d449d387, regionState=OPEN, openSeqNum=2, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:16,009 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 148d94d6f40d416655c1aa51d449d387, server=7fb8763a5d32,41809,1732521439188 because future has completed 2024-11-25T07:58:16,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-25T07:58:16,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 148d94d6f40d416655c1aa51d449d387, server=7fb8763a5d32,41809,1732521439188 in 201 msec 2024-11-25T07:58:16,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-25T07:58:16,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=148d94d6f40d416655c1aa51d449d387, ASSIGN in 366 msec 2024-11-25T07:58:16,023 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T07:58:16,024 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521496023"}]},"ts":"1732521496023"} 2024-11-25T07:58:16,027 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-25T07:58:16,028 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T07:58:16,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2210 sec 2024-11-25T07:58:16,427 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-25T07:58:17,305 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T07:58:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T07:58:17,855 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-25T07:58:17,855 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-25T07:58:17,856 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:17,863 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-25T07:58:17,863 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:17,863 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-25T07:58:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2] 2024-11-25T07:58:17,876 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T07:58:17,876 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-25T07:58:17,879 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-25T07:58:17,881 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-25T07:58:17,883 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-25T07:58:17,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-601213537888501735 2024-11-25T07:58:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46980 deadline: 1732521500882, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T07:58:17,888 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-25T07:58:17,890 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-25T07:58:17,890 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T07:58:17,892 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T07:58:17,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-601213537888501733 2024-11-25T07:58:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46980 deadline: 1732521500891, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T07:58:17,895 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T07:58:17,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-601213537888501732 2024-11-25T07:58:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46980 deadline: 1732521500894, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T07:58:17,899 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T07:58:17,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-601213537888501731 2024-11-25T07:58:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46980 deadline: 1732521500898, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T07:58:17,903 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T07:58:17,910 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 225), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=206 (was 206), ProcessCount=11 (was 11), AvailableMemoryMB=4304 (was 4321) 2024-11-25T07:58:17,917 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=206, ProcessCount=11, AvailableMemoryMB=4304 2024-11-25T07:58:17,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T07:58:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-25T07:58:17,922 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T07:58:17,922 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:17,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-25T07:58:17,923 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T07:58:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-25T07:58:17,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741843_1019 (size=418) 2024-11-25T07:58:18,334 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d7101967d1e1f9b0cf2c8406f99fcab7, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574 2024-11-25T07:58:18,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741844_1020 (size=69) 2024-11-25T07:58:18,648 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-25T07:58:18,649 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-25T07:58:18,750 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:18,750 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing d7101967d1e1f9b0cf2c8406f99fcab7, disabling compactions & flushes 2024-11-25T07:58:18,750 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. 2024-11-25T07:58:18,750 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. 2024-11-25T07:58:18,750 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. after waiting 0 ms 2024-11-25T07:58:18,750 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. 2024-11-25T07:58:18,750 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. 2024-11-25T07:58:18,750 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for d7101967d1e1f9b0cf2c8406f99fcab7: Waiting for close lock at 1732521498750Disabling compacts and flushes for region at 1732521498750Disabling writes for close at 1732521498750Writing region close event to WAL at 1732521498750Closed at 1732521498750 2024-11-25T07:58:18,752 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T07:58:18,752 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732521498752"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732521498752"}]},"ts":"1732521498752"} 2024-11-25T07:58:18,755 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T07:58:18,756 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T07:58:18,757 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521498757"}]},"ts":"1732521498757"} 2024-11-25T07:58:18,760 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-25T07:58:18,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d7101967d1e1f9b0cf2c8406f99fcab7, ASSIGN}] 2024-11-25T07:58:18,761 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d7101967d1e1f9b0cf2c8406f99fcab7, ASSIGN 2024-11-25T07:58:18,763 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d7101967d1e1f9b0cf2c8406f99fcab7, ASSIGN; state=OFFLINE, location=7fb8763a5d32,41809,1732521439188; forceNewPlan=false, retain=false 2024-11-25T07:58:18,914 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d7101967d1e1f9b0cf2c8406f99fcab7, regionState=OPENING, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:18,921 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d7101967d1e1f9b0cf2c8406f99fcab7, ASSIGN because future has completed 2024-11-25T07:58:18,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7101967d1e1f9b0cf2c8406f99fcab7, server=7fb8763a5d32,41809,1732521439188}] 2024-11-25T07:58:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-25T07:58:19,081 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. 2024-11-25T07:58:19,082 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => d7101967d1e1f9b0cf2c8406f99fcab7, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:58:19,082 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner d7101967d1e1f9b0cf2c8406f99fcab7 2024-11-25T07:58:19,082 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:19,082 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for d7101967d1e1f9b0cf2c8406f99fcab7 2024-11-25T07:58:19,082 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for d7101967d1e1f9b0cf2c8406f99fcab7 2024-11-25T07:58:19,085 INFO [StoreOpener-d7101967d1e1f9b0cf2c8406f99fcab7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d7101967d1e1f9b0cf2c8406f99fcab7 2024-11-25T07:58:19,087 INFO [StoreOpener-d7101967d1e1f9b0cf2c8406f99fcab7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7101967d1e1f9b0cf2c8406f99fcab7 columnFamilyName testFamily 2024-11-25T07:58:19,087 DEBUG [StoreOpener-d7101967d1e1f9b0cf2c8406f99fcab7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:19,088 INFO [StoreOpener-d7101967d1e1f9b0cf2c8406f99fcab7-1 {}] regionserver.HStore(327): Store=d7101967d1e1f9b0cf2c8406f99fcab7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:58:19,088 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for d7101967d1e1f9b0cf2c8406f99fcab7 2024-11-25T07:58:19,089 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScanner/d7101967d1e1f9b0cf2c8406f99fcab7 2024-11-25T07:58:19,090 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScanner/d7101967d1e1f9b0cf2c8406f99fcab7 2024-11-25T07:58:19,091 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for d7101967d1e1f9b0cf2c8406f99fcab7 2024-11-25T07:58:19,091 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for d7101967d1e1f9b0cf2c8406f99fcab7 2024-11-25T07:58:19,094 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for d7101967d1e1f9b0cf2c8406f99fcab7 2024-11-25T07:58:19,098 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScanner/d7101967d1e1f9b0cf2c8406f99fcab7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:58:19,099 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened d7101967d1e1f9b0cf2c8406f99fcab7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67484474, jitterRate=0.005597025156021118}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:58:19,099 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d7101967d1e1f9b0cf2c8406f99fcab7 2024-11-25T07:58:19,100 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for d7101967d1e1f9b0cf2c8406f99fcab7: Running coprocessor pre-open hook at 1732521499082Writing region info on filesystem at 1732521499082Initializing all the Stores at 1732521499084 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521499084Cleaning up temporary data from old regions at 1732521499091 (+7 ms)Running coprocessor post-open hooks at 1732521499099 (+8 ms)Region opened successfully at 1732521499099 2024-11-25T07:58:19,101 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7., pid=18, masterSystemTime=1732521499076 2024-11-25T07:58:19,104 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. 2024-11-25T07:58:19,104 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. 2024-11-25T07:58:19,105 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d7101967d1e1f9b0cf2c8406f99fcab7, regionState=OPEN, openSeqNum=2, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:19,108 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7101967d1e1f9b0cf2c8406f99fcab7, server=7fb8763a5d32,41809,1732521439188 because future has completed 2024-11-25T07:58:19,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-25T07:58:19,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure d7101967d1e1f9b0cf2c8406f99fcab7, server=7fb8763a5d32,41809,1732521439188 in 188 msec 2024-11-25T07:58:19,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-25T07:58:19,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d7101967d1e1f9b0cf2c8406f99fcab7, ASSIGN in 354 msec 2024-11-25T07:58:19,119 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T07:58:19,119 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521499119"}]},"ts":"1732521499119"} 2024-11-25T07:58:19,122 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-25T07:58:19,123 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T07:58:19,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2040 sec 2024-11-25T07:58:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-25T07:58:20,948 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-25T07:58:20,948 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-25T07:58:20,949 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:20,957 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-25T07:58:20,958 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:20,958 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-25T07:58:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2] 2024-11-25T07:58:20,972 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T07:58:20,972 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-25T07:58:20,975 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-25T07:58:20,977 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-25T07:58:21,476 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T07:58:21,984 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=7fb8763a5d32:41809 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-25T07:58:21,984 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-25T07:58:21,984 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 from cache 2024-11-25T07:58:22,429 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-25T07:58:22,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-25T07:58:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2] 2024-11-25T07:58:22,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-25T07:58:23,984 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=7fb8763a5d32:41809 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=987ms, rpcTimeout=978ms 2024-11-25T07:58:23,985 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=987ms, rpcTimeout=978ms 2024-11-25T07:58:23,985 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 from cache 2024-11-25T07:58:23,995 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=197 (was 206), ProcessCount=11 (was 11), AvailableMemoryMB=5625 (was 4304) - AvailableMemoryMB LEAK? - 2024-11-25T07:58:24,005 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=197, ProcessCount=11, AvailableMemoryMB=5624 2024-11-25T07:58:24,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T07:58:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-25T07:58:24,013 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T07:58:24,014 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:24,014 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-25T07:58:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T07:58:24,015 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T07:58:24,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741845_1021 (size=416) 2024-11-25T07:58:24,032 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f1cec251a05357f75aa8201c9d6696a0, NAME => 'testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574 2024-11-25T07:58:24,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741846_1022 (size=67) 2024-11-25T07:58:24,443 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:24,443 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing f1cec251a05357f75aa8201c9d6696a0, disabling compactions & flushes 2024-11-25T07:58:24,443 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. 2024-11-25T07:58:24,443 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. 2024-11-25T07:58:24,443 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. after waiting 0 ms 2024-11-25T07:58:24,443 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. 2024-11-25T07:58:24,444 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. 2024-11-25T07:58:24,444 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for f1cec251a05357f75aa8201c9d6696a0: Waiting for close lock at 1732521504443Disabling compacts and flushes for region at 1732521504443Disabling writes for close at 1732521504443Writing region close event to WAL at 1732521504443Closed at 1732521504443 2024-11-25T07:58:24,446 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T07:58:24,446 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732521504446"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732521504446"}]},"ts":"1732521504446"} 2024-11-25T07:58:24,453 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T07:58:24,454 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T07:58:24,455 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521504455"}]},"ts":"1732521504455"} 2024-11-25T07:58:24,457 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-25T07:58:24,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f1cec251a05357f75aa8201c9d6696a0, ASSIGN}] 2024-11-25T07:58:24,460 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f1cec251a05357f75aa8201c9d6696a0, ASSIGN 2024-11-25T07:58:24,462 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f1cec251a05357f75aa8201c9d6696a0, ASSIGN; state=OFFLINE, location=7fb8763a5d32,41809,1732521439188; forceNewPlan=false, retain=false 2024-11-25T07:58:24,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-25T07:58:24,613 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=f1cec251a05357f75aa8201c9d6696a0, regionState=OPENING, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:24,617 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f1cec251a05357f75aa8201c9d6696a0, ASSIGN because future has completed 2024-11-25T07:58:24,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1cec251a05357f75aa8201c9d6696a0, server=7fb8763a5d32,41809,1732521439188}] 2024-11-25T07:58:24,777 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. 2024-11-25T07:58:24,777 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => f1cec251a05357f75aa8201c9d6696a0, NAME => 'testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:58:24,778 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync f1cec251a05357f75aa8201c9d6696a0 2024-11-25T07:58:24,778 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:24,778 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for f1cec251a05357f75aa8201c9d6696a0 2024-11-25T07:58:24,778 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for f1cec251a05357f75aa8201c9d6696a0 2024-11-25T07:58:24,780 INFO [StoreOpener-f1cec251a05357f75aa8201c9d6696a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f1cec251a05357f75aa8201c9d6696a0 2024-11-25T07:58:24,783 INFO [StoreOpener-f1cec251a05357f75aa8201c9d6696a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1cec251a05357f75aa8201c9d6696a0 columnFamilyName testFamily 2024-11-25T07:58:24,783 DEBUG [StoreOpener-f1cec251a05357f75aa8201c9d6696a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:24,783 INFO [StoreOpener-f1cec251a05357f75aa8201c9d6696a0-1 {}] regionserver.HStore(327): Store=f1cec251a05357f75aa8201c9d6696a0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:58:24,784 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for f1cec251a05357f75aa8201c9d6696a0 2024-11-25T07:58:24,785 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNextAsync/f1cec251a05357f75aa8201c9d6696a0 2024-11-25T07:58:24,786 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNextAsync/f1cec251a05357f75aa8201c9d6696a0 2024-11-25T07:58:24,787 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for f1cec251a05357f75aa8201c9d6696a0 2024-11-25T07:58:24,787 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for f1cec251a05357f75aa8201c9d6696a0 2024-11-25T07:58:24,790 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for f1cec251a05357f75aa8201c9d6696a0 2024-11-25T07:58:24,793 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNextAsync/f1cec251a05357f75aa8201c9d6696a0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:58:24,794 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened f1cec251a05357f75aa8201c9d6696a0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65827285, jitterRate=-0.019097015261650085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:58:24,794 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1cec251a05357f75aa8201c9d6696a0 2024-11-25T07:58:24,795 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for f1cec251a05357f75aa8201c9d6696a0: Running coprocessor pre-open hook at 1732521504778Writing region info on filesystem at 1732521504778Initializing all the Stores at 1732521504780 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521504780Cleaning up temporary data from old regions at 1732521504787 (+7 ms)Running coprocessor post-open hooks at 1732521504794 (+7 ms)Region opened successfully at 1732521504795 (+1 ms) 2024-11-25T07:58:24,796 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0., pid=21, masterSystemTime=1732521504771 2024-11-25T07:58:24,800 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. 2024-11-25T07:58:24,800 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. 2024-11-25T07:58:24,801 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=f1cec251a05357f75aa8201c9d6696a0, regionState=OPEN, openSeqNum=2, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:24,804 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1cec251a05357f75aa8201c9d6696a0, server=7fb8763a5d32,41809,1732521439188 because future has completed 2024-11-25T07:58:24,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-25T07:58:24,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure f1cec251a05357f75aa8201c9d6696a0, server=7fb8763a5d32,41809,1732521439188 in 187 msec 2024-11-25T07:58:24,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-25T07:58:24,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f1cec251a05357f75aa8201c9d6696a0, ASSIGN in 351 msec 2024-11-25T07:58:24,815 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T07:58:24,815 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521504815"}]},"ts":"1732521504815"} 2024-11-25T07:58:24,818 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-25T07:58:24,819 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T07:58:24,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 811 msec 2024-11-25T07:58:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T07:58:25,025 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-25T07:58:25,025 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-25T07:58:25,025 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:25,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-25T07:58:25,030 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:25,030 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-25T07:58:25,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2] 2024-11-25T07:58:25,044 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T07:58:25,044 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-25T07:58:25,047 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-25T07:58:25,049 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-25T07:58:25,050 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-25T07:58:25,050 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-25T07:58:28,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-601213537888501720' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-25T07:58:28,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41809 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:46980 deadline: 1732521509060, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-601213537888501720' does not exist 2024-11-25T07:58:28,062 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 7fb8763a5d32,41809,1732521439188 for closing scanner id = -601213537888501720 for f1cec251a05357f75aa8201c9d6696a0 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-601213537888501720' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:58:28,067 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-727370367_22 at /127.0.0.1:53328 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=206 (was 197) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5293 (was 5624) 2024-11-25T07:58:28,076 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=206, ProcessCount=11, AvailableMemoryMB=5291 2024-11-25T07:58:28,078 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T07:58:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-25T07:58:28,082 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T07:58:28,082 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:28,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-25T07:58:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-25T07:58:28,084 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T07:58:28,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741847_1023 (size=411) 2024-11-25T07:58:28,431 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-25T07:58:28,495 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 908344ba8601916f4c690fceca67003a, NAME => 'testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574 2024-11-25T07:58:28,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741848_1024 (size=62) 2024-11-25T07:58:28,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-25T07:58:28,649 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-25T07:58:28,649 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-25T07:58:28,650 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-25T07:58:28,650 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T07:58:28,904 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:28,904 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 908344ba8601916f4c690fceca67003a, disabling compactions & flushes 2024-11-25T07:58:28,904 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. 2024-11-25T07:58:28,904 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. 2024-11-25T07:58:28,904 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. after waiting 0 ms 2024-11-25T07:58:28,904 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. 2024-11-25T07:58:28,904 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. 2024-11-25T07:58:28,904 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 908344ba8601916f4c690fceca67003a: Waiting for close lock at 1732521508904Disabling compacts and flushes for region at 1732521508904Disabling writes for close at 1732521508904Writing region close event to WAL at 1732521508904Closed at 1732521508904 2024-11-25T07:58:28,906 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T07:58:28,906 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732521508906"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732521508906"}]},"ts":"1732521508906"} 2024-11-25T07:58:28,909 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T07:58:28,910 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T07:58:28,910 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521508910"}]},"ts":"1732521508910"} 2024-11-25T07:58:28,913 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-25T07:58:28,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=908344ba8601916f4c690fceca67003a, ASSIGN}] 2024-11-25T07:58:28,966 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=908344ba8601916f4c690fceca67003a, ASSIGN 2024-11-25T07:58:28,967 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=908344ba8601916f4c690fceca67003a, ASSIGN; state=OFFLINE, location=7fb8763a5d32,41809,1732521439188; forceNewPlan=false, retain=false 2024-11-25T07:58:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-25T07:58:29,119 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=908344ba8601916f4c690fceca67003a, regionState=OPENING, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:29,124 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=908344ba8601916f4c690fceca67003a, ASSIGN because future has completed 2024-11-25T07:58:29,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 908344ba8601916f4c690fceca67003a, server=7fb8763a5d32,41809,1732521439188}] 2024-11-25T07:58:29,282 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. 2024-11-25T07:58:29,283 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 908344ba8601916f4c690fceca67003a, NAME => 'testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:58:29,283 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 908344ba8601916f4c690fceca67003a 2024-11-25T07:58:29,283 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:29,283 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 908344ba8601916f4c690fceca67003a 2024-11-25T07:58:29,283 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 908344ba8601916f4c690fceca67003a 2024-11-25T07:58:29,285 INFO [StoreOpener-908344ba8601916f4c690fceca67003a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 908344ba8601916f4c690fceca67003a 2024-11-25T07:58:29,287 INFO [StoreOpener-908344ba8601916f4c690fceca67003a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 908344ba8601916f4c690fceca67003a columnFamilyName testFamily 2024-11-25T07:58:29,287 DEBUG [StoreOpener-908344ba8601916f4c690fceca67003a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:29,288 INFO [StoreOpener-908344ba8601916f4c690fceca67003a-1 {}] regionserver.HStore(327): Store=908344ba8601916f4c690fceca67003a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:58:29,288 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 908344ba8601916f4c690fceca67003a 2024-11-25T07:58:29,290 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNext/908344ba8601916f4c690fceca67003a 2024-11-25T07:58:29,290 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNext/908344ba8601916f4c690fceca67003a 2024-11-25T07:58:29,291 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 908344ba8601916f4c690fceca67003a 2024-11-25T07:58:29,291 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 908344ba8601916f4c690fceca67003a 2024-11-25T07:58:29,293 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 908344ba8601916f4c690fceca67003a 2024-11-25T07:58:29,297 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNext/908344ba8601916f4c690fceca67003a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:58:29,298 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 908344ba8601916f4c690fceca67003a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59798840, jitterRate=-0.10892784595489502}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:58:29,298 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 908344ba8601916f4c690fceca67003a 2024-11-25T07:58:29,298 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 908344ba8601916f4c690fceca67003a: Running coprocessor pre-open hook at 1732521509283Writing region info on filesystem at 1732521509283Initializing all the Stores at 1732521509285 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521509285Cleaning up temporary data from old regions at 1732521509291 (+6 ms)Running coprocessor post-open hooks at 1732521509298 (+7 ms)Region opened successfully at 1732521509298 2024-11-25T07:58:29,300 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a., pid=24, masterSystemTime=1732521509277 2024-11-25T07:58:29,303 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. 2024-11-25T07:58:29,303 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. 2024-11-25T07:58:29,304 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=908344ba8601916f4c690fceca67003a, regionState=OPEN, openSeqNum=2, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:29,306 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 908344ba8601916f4c690fceca67003a, server=7fb8763a5d32,41809,1732521439188 because future has completed 2024-11-25T07:58:29,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-25T07:58:29,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 908344ba8601916f4c690fceca67003a, server=7fb8763a5d32,41809,1732521439188 in 184 msec 2024-11-25T07:58:29,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-25T07:58:29,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=908344ba8601916f4c690fceca67003a, ASSIGN in 398 msec 2024-11-25T07:58:29,316 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T07:58:29,317 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521509317"}]},"ts":"1732521509317"} 2024-11-25T07:58:29,320 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-25T07:58:29,321 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T07:58:29,324 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2440 sec 2024-11-25T07:58:30,284 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T07:58:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-25T07:58:31,105 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-25T07:58:31,105 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-25T07:58:31,105 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:31,113 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-25T07:58:31,113 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:31,113 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-25T07:58:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2] 2024-11-25T07:58:31,126 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T07:58:31,126 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-25T07:58:31,129 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-25T07:58:31,131 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-25T07:58:31,132 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-25T07:58:31,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-25T07:58:33,223 INFO [master/7fb8763a5d32:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-25T07:58:33,224 INFO [master/7fb8763a5d32:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-25T07:58:34,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-601213537888501716' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-25T07:58:34,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41809 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:46980 deadline: 1732521515137, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-601213537888501716' does not exist 2024-11-25T07:58:34,138 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 7fb8763a5d32,41809,1732521439188 for closing scanner id = -601213537888501716 for 908344ba8601916f4c690fceca67003a of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-601213537888501716' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:58:34,142 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 223), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=189 (was 206), ProcessCount=11 (was 11), AvailableMemoryMB=5078 (was 5291) 2024-11-25T07:58:34,149 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=189, ProcessCount=11, AvailableMemoryMB=5078 2024-11-25T07:58:34,151 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35737 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T07:58:34,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35737 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-25T07:58:34,154 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T07:58:34,154 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:34,154 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35737 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-25T07:58:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T07:58:34,155 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T07:58:34,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741849_1025 (size=419) 2024-11-25T07:58:34,433 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-25T07:58:34,569 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b412ace892c85f02a9cc16735a0cce4c, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574 2024-11-25T07:58:34,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741850_1026 (size=70) 2024-11-25T07:58:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-25T07:58:34,982 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:34,982 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing b412ace892c85f02a9cc16735a0cce4c, disabling compactions & flushes 2024-11-25T07:58:34,982 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. 2024-11-25T07:58:34,982 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. 2024-11-25T07:58:34,982 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. after waiting 0 ms 2024-11-25T07:58:34,983 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. 2024-11-25T07:58:34,983 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. 2024-11-25T07:58:34,983 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for b412ace892c85f02a9cc16735a0cce4c: Waiting for close lock at 1732521514982Disabling compacts and flushes for region at 1732521514982Disabling writes for close at 1732521514983 (+1 ms)Writing region close event to WAL at 1732521514983Closed at 1732521514983 2024-11-25T07:58:34,986 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T07:58:34,987 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732521514986"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732521514986"}]},"ts":"1732521514986"} 2024-11-25T07:58:34,993 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T07:58:34,995 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T07:58:34,995 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521514995"}]},"ts":"1732521514995"} 2024-11-25T07:58:34,997 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-25T07:58:34,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=b412ace892c85f02a9cc16735a0cce4c, ASSIGN}] 2024-11-25T07:58:34,999 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=b412ace892c85f02a9cc16735a0cce4c, ASSIGN 2024-11-25T07:58:35,000 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=b412ace892c85f02a9cc16735a0cce4c, ASSIGN; state=OFFLINE, location=7fb8763a5d32,41809,1732521439188; forceNewPlan=false, retain=false 2024-11-25T07:58:35,151 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=b412ace892c85f02a9cc16735a0cce4c, regionState=OPENING, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:35,154 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=b412ace892c85f02a9cc16735a0cce4c, ASSIGN because future has completed 2024-11-25T07:58:35,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure b412ace892c85f02a9cc16735a0cce4c, server=7fb8763a5d32,41809,1732521439188}] 2024-11-25T07:58:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T07:58:35,312 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. 2024-11-25T07:58:35,312 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => b412ace892c85f02a9cc16735a0cce4c, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:58:35,313 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync b412ace892c85f02a9cc16735a0cce4c 2024-11-25T07:58:35,313 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:35,313 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for b412ace892c85f02a9cc16735a0cce4c 2024-11-25T07:58:35,313 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for b412ace892c85f02a9cc16735a0cce4c 2024-11-25T07:58:35,316 INFO [StoreOpener-b412ace892c85f02a9cc16735a0cce4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b412ace892c85f02a9cc16735a0cce4c 2024-11-25T07:58:35,319 INFO [StoreOpener-b412ace892c85f02a9cc16735a0cce4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b412ace892c85f02a9cc16735a0cce4c columnFamilyName testFamily 2024-11-25T07:58:35,319 DEBUG [StoreOpener-b412ace892c85f02a9cc16735a0cce4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:35,320 INFO [StoreOpener-b412ace892c85f02a9cc16735a0cce4c-1 {}] regionserver.HStore(327): Store=b412ace892c85f02a9cc16735a0cce4c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:58:35,320 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for b412ace892c85f02a9cc16735a0cce4c 2024-11-25T07:58:35,321 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b412ace892c85f02a9cc16735a0cce4c 2024-11-25T07:58:35,322 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b412ace892c85f02a9cc16735a0cce4c 2024-11-25T07:58:35,322 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for b412ace892c85f02a9cc16735a0cce4c 2024-11-25T07:58:35,322 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for b412ace892c85f02a9cc16735a0cce4c 2024-11-25T07:58:35,325 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for b412ace892c85f02a9cc16735a0cce4c 2024-11-25T07:58:35,327 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b412ace892c85f02a9cc16735a0cce4c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:58:35,328 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened b412ace892c85f02a9cc16735a0cce4c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68481757, jitterRate=0.02045769989490509}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:58:35,328 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b412ace892c85f02a9cc16735a0cce4c 2024-11-25T07:58:35,329 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for b412ace892c85f02a9cc16735a0cce4c: Running coprocessor pre-open hook at 1732521515313Writing region info on filesystem at 1732521515313Initializing all the Stores at 1732521515316 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521515316Cleaning up temporary data from old regions at 1732521515322 (+6 ms)Running coprocessor post-open hooks at 1732521515328 (+6 ms)Region opened successfully at 1732521515329 (+1 ms) 2024-11-25T07:58:35,330 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c., pid=27, masterSystemTime=1732521515306 2024-11-25T07:58:35,333 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. 2024-11-25T07:58:35,333 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. 2024-11-25T07:58:35,333 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=b412ace892c85f02a9cc16735a0cce4c, regionState=OPEN, openSeqNum=2, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:35,336 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure b412ace892c85f02a9cc16735a0cce4c, server=7fb8763a5d32,41809,1732521439188 because future has completed 2024-11-25T07:58:35,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-25T07:58:35,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure b412ace892c85f02a9cc16735a0cce4c, server=7fb8763a5d32,41809,1732521439188 in 183 msec 2024-11-25T07:58:35,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-25T07:58:35,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=b412ace892c85f02a9cc16735a0cce4c, ASSIGN in 343 msec 2024-11-25T07:58:35,346 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T07:58:35,347 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521515346"}]},"ts":"1732521515346"} 2024-11-25T07:58:35,350 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-25T07:58:35,352 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T07:58:35,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2010 sec 2024-11-25T07:58:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T07:58:37,185 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-25T07:58:37,185 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-25T07:58:37,185 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:37,194 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-25T07:58:37,194 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:37,194 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-25T07:58:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2] 2024-11-25T07:58:37,207 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T07:58:37,208 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-25T07:58:37,210 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-25T07:58:37,212 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-25T07:58:38,649 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-25T07:58:38,649 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-25T07:58:38,650 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-25T07:58:38,650 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T07:58:40,850 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T07:58:40,874 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-25T07:58:43,214 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=7fb8763a5d32:41809 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6003ms, rpcTimeout=5999ms 2024-11-25T07:58:43,215 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6003ms, rpcTimeout=5999ms 2024-11-25T07:58:43,215 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 from cache 2024-11-25T07:58:43,229 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=160 (was 189), ProcessCount=11 (was 11), AvailableMemoryMB=5002 (was 5078) 2024-11-25T07:58:43,239 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=160, ProcessCount=11, AvailableMemoryMB=5002 2024-11-25T07:58:43,242 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35737 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T07:58:43,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35737 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-25T07:58:43,245 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T07:58:43,245 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:43,245 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35737 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-25T07:58:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-25T07:58:43,246 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T07:58:43,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741851_1027 (size=407) 2024-11-25T07:58:43,257 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6e414485380d7d765d4f3e076fcdb9f1, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574 2024-11-25T07:58:43,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741852_1028 (size=58) 2024-11-25T07:58:43,667 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:43,667 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 6e414485380d7d765d4f3e076fcdb9f1, disabling compactions & flushes 2024-11-25T07:58:43,668 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. 2024-11-25T07:58:43,668 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. 2024-11-25T07:58:43,668 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. after waiting 0 ms 2024-11-25T07:58:43,668 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. 2024-11-25T07:58:43,668 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. 2024-11-25T07:58:43,668 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6e414485380d7d765d4f3e076fcdb9f1: Waiting for close lock at 1732521523667Disabling compacts and flushes for region at 1732521523667Disabling writes for close at 1732521523668 (+1 ms)Writing region close event to WAL at 1732521523668Closed at 1732521523668 2024-11-25T07:58:43,670 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T07:58:43,671 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732521523671"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732521523671"}]},"ts":"1732521523671"} 2024-11-25T07:58:43,673 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T07:58:43,675 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T07:58:43,675 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521523675"}]},"ts":"1732521523675"} 2024-11-25T07:58:43,677 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-25T07:58:43,678 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=6e414485380d7d765d4f3e076fcdb9f1, ASSIGN}] 2024-11-25T07:58:43,679 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=6e414485380d7d765d4f3e076fcdb9f1, ASSIGN 2024-11-25T07:58:43,681 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=6e414485380d7d765d4f3e076fcdb9f1, ASSIGN; state=OFFLINE, location=7fb8763a5d32,41809,1732521439188; forceNewPlan=false, retain=false 2024-11-25T07:58:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-25T07:58:43,832 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=6e414485380d7d765d4f3e076fcdb9f1, regionState=OPENING, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:43,836 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=6e414485380d7d765d4f3e076fcdb9f1, ASSIGN because future has completed 2024-11-25T07:58:43,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e414485380d7d765d4f3e076fcdb9f1, server=7fb8763a5d32,41809,1732521439188}] 2024-11-25T07:58:43,999 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. 2024-11-25T07:58:44,000 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e414485380d7d765d4f3e076fcdb9f1, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:58:44,001 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 6e414485380d7d765d4f3e076fcdb9f1 2024-11-25T07:58:44,001 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:44,001 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 6e414485380d7d765d4f3e076fcdb9f1 2024-11-25T07:58:44,001 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 6e414485380d7d765d4f3e076fcdb9f1 2024-11-25T07:58:44,003 INFO [StoreOpener-6e414485380d7d765d4f3e076fcdb9f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6e414485380d7d765d4f3e076fcdb9f1 2024-11-25T07:58:44,006 INFO [StoreOpener-6e414485380d7d765d4f3e076fcdb9f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e414485380d7d765d4f3e076fcdb9f1 columnFamilyName testFamily 2024-11-25T07:58:44,006 DEBUG [StoreOpener-6e414485380d7d765d4f3e076fcdb9f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:44,007 INFO [StoreOpener-6e414485380d7d765d4f3e076fcdb9f1-1 {}] regionserver.HStore(327): Store=6e414485380d7d765d4f3e076fcdb9f1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:58:44,007 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 6e414485380d7d765d4f3e076fcdb9f1 2024-11-25T07:58:44,008 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNext/6e414485380d7d765d4f3e076fcdb9f1 2024-11-25T07:58:44,009 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNext/6e414485380d7d765d4f3e076fcdb9f1 2024-11-25T07:58:44,009 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 6e414485380d7d765d4f3e076fcdb9f1 2024-11-25T07:58:44,009 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 6e414485380d7d765d4f3e076fcdb9f1 2024-11-25T07:58:44,011 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 6e414485380d7d765d4f3e076fcdb9f1 2024-11-25T07:58:44,014 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNext/6e414485380d7d765d4f3e076fcdb9f1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:58:44,014 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 6e414485380d7d765d4f3e076fcdb9f1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61587449, jitterRate=-0.08227549493312836}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:58:44,014 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e414485380d7d765d4f3e076fcdb9f1 2024-11-25T07:58:44,015 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 6e414485380d7d765d4f3e076fcdb9f1: Running coprocessor pre-open hook at 1732521524001Writing region info on filesystem at 1732521524001Initializing all the Stores at 1732521524003 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521524003Cleaning up temporary data from old regions at 1732521524009 (+6 ms)Running coprocessor post-open hooks at 1732521524014 (+5 ms)Region opened successfully at 1732521524015 (+1 ms) 2024-11-25T07:58:44,016 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1., pid=30, masterSystemTime=1732521523991 2024-11-25T07:58:44,019 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. 2024-11-25T07:58:44,019 INFO [RS_OPEN_PRIORITY_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. 2024-11-25T07:58:44,019 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=6e414485380d7d765d4f3e076fcdb9f1, regionState=OPEN, openSeqNum=2, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:44,022 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e414485380d7d765d4f3e076fcdb9f1, server=7fb8763a5d32,41809,1732521439188 because future has completed 2024-11-25T07:58:44,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-25T07:58:44,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e414485380d7d765d4f3e076fcdb9f1, server=7fb8763a5d32,41809,1732521439188 in 186 msec 2024-11-25T07:58:44,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-25T07:58:44,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=6e414485380d7d765d4f3e076fcdb9f1, ASSIGN in 347 msec 2024-11-25T07:58:44,029 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T07:58:44,029 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521524029"}]},"ts":"1732521524029"} 2024-11-25T07:58:44,032 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-25T07:58:44,033 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T07:58:44,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 791 msec 2024-11-25T07:58:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-25T07:58:44,257 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-25T07:58:44,258 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-25T07:58:44,259 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:44,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-25T07:58:44,269 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:44,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-25T07:58:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2] 2024-11-25T07:58:44,286 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T07:58:44,287 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-25T07:58:44,291 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-25T07:58:44,293 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-25T07:58:44,295 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-25T07:58:44,295 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-25T07:58:46,876 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-25T07:58:47,306 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T07:58:48,649 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-25T07:58:48,649 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-25T07:58:49,505 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T07:58:50,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-601213537888501708' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-25T07:58:50,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41809 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:46980 deadline: 1732521536308, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-601213537888501708' does not exist 2024-11-25T07:58:50,309 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 7fb8763a5d32,41809,1732521439188 for closing scanner id = -601213537888501708 for 6e414485380d7d765d4f3e076fcdb9f1 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-601213537888501708' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T07:58:50,318 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=174 (was 160) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4842 (was 5002) 2024-11-25T07:58:50,326 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=174, ProcessCount=11, AvailableMemoryMB=4841 2024-11-25T07:58:50,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T07:58:50,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-25T07:58:50,330 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T07:58:50,331 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:50,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-25T07:58:50,332 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T07:58:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T07:58:50,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741853_1029 (size=423) 2024-11-25T07:58:50,743 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 239239bec824bf759590c29a95c9a1b2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574 2024-11-25T07:58:50,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741854_1030 (size=74) 2024-11-25T07:58:50,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-25T07:58:51,155 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:51,156 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 239239bec824bf759590c29a95c9a1b2, disabling compactions & flushes 2024-11-25T07:58:51,156 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. 2024-11-25T07:58:51,156 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. 2024-11-25T07:58:51,156 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. after waiting 0 ms 2024-11-25T07:58:51,156 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. 2024-11-25T07:58:51,156 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. 2024-11-25T07:58:51,156 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 239239bec824bf759590c29a95c9a1b2: Waiting for close lock at 1732521531155Disabling compacts and flushes for region at 1732521531155Disabling writes for close at 1732521531156 (+1 ms)Writing region close event to WAL at 1732521531156Closed at 1732521531156 2024-11-25T07:58:51,159 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T07:58:51,160 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732521531160"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732521531160"}]},"ts":"1732521531160"} 2024-11-25T07:58:51,164 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T07:58:51,165 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T07:58:51,166 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521531166"}]},"ts":"1732521531166"} 2024-11-25T07:58:51,168 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-25T07:58:51,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=239239bec824bf759590c29a95c9a1b2, ASSIGN}] 2024-11-25T07:58:51,169 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=239239bec824bf759590c29a95c9a1b2, ASSIGN 2024-11-25T07:58:51,170 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=239239bec824bf759590c29a95c9a1b2, ASSIGN; state=OFFLINE, location=7fb8763a5d32,41809,1732521439188; forceNewPlan=false, retain=false 2024-11-25T07:58:51,321 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=239239bec824bf759590c29a95c9a1b2, regionState=OPENING, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:51,325 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=239239bec824bf759590c29a95c9a1b2, ASSIGN because future has completed 2024-11-25T07:58:51,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 239239bec824bf759590c29a95c9a1b2, server=7fb8763a5d32,41809,1732521439188}] 2024-11-25T07:58:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T07:58:51,486 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. 2024-11-25T07:58:51,487 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 239239bec824bf759590c29a95c9a1b2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2.', STARTKEY => '', ENDKEY => ''} 2024-11-25T07:58:51,488 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 239239bec824bf759590c29a95c9a1b2 2024-11-25T07:58:51,488 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T07:58:51,488 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 239239bec824bf759590c29a95c9a1b2 2024-11-25T07:58:51,489 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 239239bec824bf759590c29a95c9a1b2 2024-11-25T07:58:51,492 INFO [StoreOpener-239239bec824bf759590c29a95c9a1b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 239239bec824bf759590c29a95c9a1b2 2024-11-25T07:58:51,493 INFO [StoreOpener-239239bec824bf759590c29a95c9a1b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 239239bec824bf759590c29a95c9a1b2 columnFamilyName testFamily 2024-11-25T07:58:51,494 DEBUG [StoreOpener-239239bec824bf759590c29a95c9a1b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T07:58:51,494 INFO [StoreOpener-239239bec824bf759590c29a95c9a1b2-1 {}] regionserver.HStore(327): Store=239239bec824bf759590c29a95c9a1b2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T07:58:51,494 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 239239bec824bf759590c29a95c9a1b2 2024-11-25T07:58:51,495 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScannerAsync/239239bec824bf759590c29a95c9a1b2 2024-11-25T07:58:51,496 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScannerAsync/239239bec824bf759590c29a95c9a1b2 2024-11-25T07:58:51,496 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 239239bec824bf759590c29a95c9a1b2 2024-11-25T07:58:51,496 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 239239bec824bf759590c29a95c9a1b2 2024-11-25T07:58:51,498 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 239239bec824bf759590c29a95c9a1b2 2024-11-25T07:58:51,501 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScannerAsync/239239bec824bf759590c29a95c9a1b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T07:58:51,502 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 239239bec824bf759590c29a95c9a1b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72531633, jitterRate=0.08080555498600006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T07:58:51,502 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 239239bec824bf759590c29a95c9a1b2 2024-11-25T07:58:51,503 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 239239bec824bf759590c29a95c9a1b2: Running coprocessor pre-open hook at 1732521531489Writing region info on filesystem at 1732521531489Initializing all the Stores at 1732521531491 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732521531492 (+1 ms)Cleaning up temporary data from old regions at 1732521531496 (+4 ms)Running coprocessor post-open hooks at 1732521531502 (+6 ms)Region opened successfully at 1732521531502 2024-11-25T07:58:51,504 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., pid=33, masterSystemTime=1732521531477 2024-11-25T07:58:51,506 DEBUG [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. 2024-11-25T07:58:51,506 INFO [RS_OPEN_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. 2024-11-25T07:58:51,507 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=239239bec824bf759590c29a95c9a1b2, regionState=OPEN, openSeqNum=2, regionLocation=7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:51,510 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 239239bec824bf759590c29a95c9a1b2, server=7fb8763a5d32,41809,1732521439188 because future has completed 2024-11-25T07:58:51,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-25T07:58:51,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 239239bec824bf759590c29a95c9a1b2, server=7fb8763a5d32,41809,1732521439188 in 186 msec 2024-11-25T07:58:51,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-25T07:58:51,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=239239bec824bf759590c29a95c9a1b2, ASSIGN in 346 msec 2024-11-25T07:58:51,517 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T07:58:51,517 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732521531517"}]},"ts":"1732521531517"} 2024-11-25T07:58:51,520 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-25T07:58:51,521 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T07:58:51,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1930 sec 2024-11-25T07:58:52,880 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-25T07:58:53,173 DEBUG [master/7fb8763a5d32:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T07:58:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T07:58:53,357 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-25T07:58:53,357 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-25T07:58:53,358 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:53,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-25T07:58:53,368 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T07:58:53,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-25T07:58:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2] 2024-11-25T07:58:53,387 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T07:58:53,388 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-25T07:58:53,391 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-25T07:58:53,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-25T07:58:54,394 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=7fb8763a5d32:41809 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-25T07:58:54,395 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-25T07:58:54,395 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 from cache 2024-11-25T07:58:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-25T07:58:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2] 2024-11-25T07:58:55,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41809 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-25T07:58:56,395 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=7fb8763a5d32:41809 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=987ms, rpcTimeout=984ms 2024-11-25T07:58:56,395 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=987ms, rpcTimeout=984ms 2024-11-25T07:58:56,395 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., hostname=7fb8763a5d32,41809,1732521439188, seqNum=2 from cache 2024-11-25T07:58:56,413 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=160 (was 174), ProcessCount=11 (was 11), AvailableMemoryMB=4729 (was 4841) 2024-11-25T07:58:56,413 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T07:58:56,413 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T07:58:56,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:58:56,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:58:56,418 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T07:58:56,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T07:58:56,419 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T07:58:56,419 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T07:58:56,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:58:56,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:58:56,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T07:58:56,419 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T07:58:56,419 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=331909808, stopped=false 2024-11-25T07:58:56,419 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7fb8763a5d32,35737,1732521438242 2024-11-25T07:58:56,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T07:58:56,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T07:58:56,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:58:56,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:58:56,477 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T07:58:56,477 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T07:58:56,478 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T07:58:56,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:58:56,478 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T07:58:56,479 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T07:58:56,479 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7fb8763a5d32,41809,1732521439188' ***** 2024-11-25T07:58:56,479 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T07:58:56,480 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T07:58:56,480 INFO [RS:0;7fb8763a5d32:41809 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T07:58:56,480 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T07:58:56,481 INFO [RS:0;7fb8763a5d32:41809 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T07:58:56,481 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(3091): Received CLOSE for e450be747e11bc58612b8b5baa075bfa 2024-11-25T07:58:56,482 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(3091): Received CLOSE for d7101967d1e1f9b0cf2c8406f99fcab7 2024-11-25T07:58:56,482 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(3091): Received CLOSE for b412ace892c85f02a9cc16735a0cce4c 2024-11-25T07:58:56,482 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(3091): Received CLOSE for f1cec251a05357f75aa8201c9d6696a0 2024-11-25T07:58:56,483 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(3091): Received CLOSE for 6e414485380d7d765d4f3e076fcdb9f1 2024-11-25T07:58:56,483 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(3091): Received CLOSE for 908344ba8601916f4c690fceca67003a 2024-11-25T07:58:56,483 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(3091): Received CLOSE for 470929e933e6a8a588c3fc1fd5ba5065 2024-11-25T07:58:56,483 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(3091): Received CLOSE for ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:56,483 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(3091): Received CLOSE for 239239bec824bf759590c29a95c9a1b2 2024-11-25T07:58:56,483 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e450be747e11bc58612b8b5baa075bfa, disabling compactions & flushes 2024-11-25T07:58:56,483 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(3091): Received CLOSE for 148d94d6f40d416655c1aa51d449d387 2024-11-25T07:58:56,483 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. 2024-11-25T07:58:56,483 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(959): stopping server 7fb8763a5d32,41809,1732521439188 2024-11-25T07:58:56,484 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. 2024-11-25T07:58:56,484 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T07:58:56,484 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. after waiting 0 ms 2024-11-25T07:58:56,484 INFO [RS:0;7fb8763a5d32:41809 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7fb8763a5d32:41809. 2024-11-25T07:58:56,484 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. 2024-11-25T07:58:56,484 DEBUG [RS:0;7fb8763a5d32:41809 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T07:58:56,484 DEBUG [RS:0;7fb8763a5d32:41809 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:58:56,484 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T07:58:56,484 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T07:58:56,484 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T07:58:56,484 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T07:58:56,485 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-25T07:58:56,485 DEBUG [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T07:58:56,485 INFO [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T07:58:56,485 DEBUG [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T07:58:56,485 DEBUG [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T07:58:56,485 DEBUG [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T07:58:56,485 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 1588230740, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 6e414485380d7d765d4f3e076fcdb9f1, 908344ba8601916f4c690fceca67003a, b412ace892c85f02a9cc16735a0cce4c, d7101967d1e1f9b0cf2c8406f99fcab7, e450be747e11bc58612b8b5baa075bfa, f1cec251a05357f75aa8201c9d6696a0, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:56,488 INFO [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-25T07:58:56,488 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e450be747e11bc58612b8b5baa075bfa 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T07:58:56,546 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e450be747e11bc58612b8b5baa075bfa/.tmp/testFamily/46ae5a88f1954b428681ac92bb8b9e59 is 61, key is row-0/testFamily:testQualifier/1732521476490/Put/seqid=0 2024-11-25T07:58:56,554 DEBUG [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/.tmp/info/c39f9875041d4aa5a719b02366fbbc22 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa./info:regioninfo/1732521474651/Put/seqid=0 2024-11-25T07:58:56,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741855_1031 (size=5049) 2024-11-25T07:58:56,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741856_1032 (size=20564) 2024-11-25T07:58:56,685 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 1588230740, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 6e414485380d7d765d4f3e076fcdb9f1, 908344ba8601916f4c690fceca67003a, b412ace892c85f02a9cc16735a0cce4c, d7101967d1e1f9b0cf2c8406f99fcab7, e450be747e11bc58612b8b5baa075bfa, f1cec251a05357f75aa8201c9d6696a0, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:56,886 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 1588230740, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 6e414485380d7d765d4f3e076fcdb9f1, 908344ba8601916f4c690fceca67003a, b412ace892c85f02a9cc16735a0cce4c, d7101967d1e1f9b0cf2c8406f99fcab7, e450be747e11bc58612b8b5baa075bfa, f1cec251a05357f75aa8201c9d6696a0, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:56,909 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-25T07:58:56,958 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e450be747e11bc58612b8b5baa075bfa/.tmp/testFamily/46ae5a88f1954b428681ac92bb8b9e59 2024-11-25T07:58:56,962 INFO [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/.tmp/info/c39f9875041d4aa5a719b02366fbbc22 2024-11-25T07:58:56,989 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T07:58:57,008 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e450be747e11bc58612b8b5baa075bfa/.tmp/testFamily/46ae5a88f1954b428681ac92bb8b9e59 as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e450be747e11bc58612b8b5baa075bfa/testFamily/46ae5a88f1954b428681ac92bb8b9e59 2024-11-25T07:58:57,017 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e450be747e11bc58612b8b5baa075bfa/testFamily/46ae5a88f1954b428681ac92bb8b9e59, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T07:58:57,024 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e450be747e11bc58612b8b5baa075bfa in 536ms, sequenceid=8, compaction requested=false 2024-11-25T07:58:57,030 DEBUG [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/.tmp/ns/2ee25bd52c5645e0af690a49957f1a52 is 43, key is default/ns:d/1732521473116/Put/seqid=0 2024-11-25T07:58:57,031 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e450be747e11bc58612b8b5baa075bfa/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T07:58:57,034 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. 2024-11-25T07:58:57,034 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e450be747e11bc58612b8b5baa075bfa: Waiting for close lock at 1732521536482Running coprocessor pre-close hooks at 1732521536483 (+1 ms)Disabling compacts and flushes for region at 1732521536483Disabling writes for close at 1732521536484 (+1 ms)Obtaining lock to block concurrent updates at 1732521536488 (+4 ms)Preparing flush snapshotting stores in e450be747e11bc58612b8b5baa075bfa at 1732521536488Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732521536495 (+7 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. at 1732521536496 (+1 ms)Flushing e450be747e11bc58612b8b5baa075bfa/testFamily: creating writer at 1732521536497 (+1 ms)Flushing e450be747e11bc58612b8b5baa075bfa/testFamily: appending metadata at 1732521536539 (+42 ms)Flushing e450be747e11bc58612b8b5baa075bfa/testFamily: closing flushed file at 1732521536542 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ff57bd6: reopening flushed file at 1732521537006 (+464 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e450be747e11bc58612b8b5baa075bfa in 536ms, sequenceid=8, compaction requested=false at 1732521537024 (+18 ms)Writing region close event to WAL at 1732521537026 (+2 ms)Running coprocessor post-close hooks at 1732521537032 (+6 ms)Closed at 1732521537034 (+2 ms) 2024-11-25T07:58:57,035 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732521473373.e450be747e11bc58612b8b5baa075bfa. 2024-11-25T07:58:57,035 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d7101967d1e1f9b0cf2c8406f99fcab7, disabling compactions & flushes 2024-11-25T07:58:57,035 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. 2024-11-25T07:58:57,035 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. 2024-11-25T07:58:57,035 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. after waiting 0 ms 2024-11-25T07:58:57,035 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. 2024-11-25T07:58:57,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741857_1033 (size=5153) 2024-11-25T07:58:57,035 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d7101967d1e1f9b0cf2c8406f99fcab7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T07:58:57,054 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScanner/d7101967d1e1f9b0cf2c8406f99fcab7/.tmp/testFamily/379c2d655195422e8cee38c5299baa67 is 61, key is row-0/testFamily:testQualifier/1732521500963/Put/seqid=0 2024-11-25T07:58:57,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741858_1034 (size=5049) 2024-11-25T07:58:57,086 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 1588230740, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 6e414485380d7d765d4f3e076fcdb9f1, 908344ba8601916f4c690fceca67003a, b412ace892c85f02a9cc16735a0cce4c, d7101967d1e1f9b0cf2c8406f99fcab7, f1cec251a05357f75aa8201c9d6696a0, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:57,287 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 1588230740, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 6e414485380d7d765d4f3e076fcdb9f1, 908344ba8601916f4c690fceca67003a, b412ace892c85f02a9cc16735a0cce4c, d7101967d1e1f9b0cf2c8406f99fcab7, f1cec251a05357f75aa8201c9d6696a0, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:57,350 INFO [regionserver/7fb8763a5d32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-25T07:58:57,350 INFO [regionserver/7fb8763a5d32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-25T07:58:57,437 INFO [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/.tmp/ns/2ee25bd52c5645e0af690a49957f1a52 2024-11-25T07:58:57,461 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScanner/d7101967d1e1f9b0cf2c8406f99fcab7/.tmp/testFamily/379c2d655195422e8cee38c5299baa67 2024-11-25T07:58:57,471 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScanner/d7101967d1e1f9b0cf2c8406f99fcab7/.tmp/testFamily/379c2d655195422e8cee38c5299baa67 as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScanner/d7101967d1e1f9b0cf2c8406f99fcab7/testFamily/379c2d655195422e8cee38c5299baa67 2024-11-25T07:58:57,474 DEBUG [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/.tmp/table/de20097d1027433896f388f6ec3434f0 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732521474671/Put/seqid=0 2024-11-25T07:58:57,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741859_1035 (size=6699) 2024-11-25T07:58:57,479 INFO [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/.tmp/table/de20097d1027433896f388f6ec3434f0 2024-11-25T07:58:57,481 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScanner/d7101967d1e1f9b0cf2c8406f99fcab7/testFamily/379c2d655195422e8cee38c5299baa67, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T07:58:57,484 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d7101967d1e1f9b0cf2c8406f99fcab7 in 449ms, sequenceid=8, compaction requested=false 2024-11-25T07:58:57,487 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-11-25T07:58:57,487 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 1588230740, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 6e414485380d7d765d4f3e076fcdb9f1, 908344ba8601916f4c690fceca67003a, b412ace892c85f02a9cc16735a0cce4c, d7101967d1e1f9b0cf2c8406f99fcab7, f1cec251a05357f75aa8201c9d6696a0, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:57,488 DEBUG [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/.tmp/info/c39f9875041d4aa5a719b02366fbbc22 as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/info/c39f9875041d4aa5a719b02366fbbc22 2024-11-25T07:58:57,490 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScanner/d7101967d1e1f9b0cf2c8406f99fcab7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T07:58:57,491 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. 2024-11-25T07:58:57,491 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d7101967d1e1f9b0cf2c8406f99fcab7: Waiting for close lock at 1732521537035Running coprocessor pre-close hooks at 1732521537035Disabling compacts and flushes for region at 1732521537035Disabling writes for close at 1732521537035Obtaining lock to block concurrent updates at 1732521537036 (+1 ms)Preparing flush snapshotting stores in d7101967d1e1f9b0cf2c8406f99fcab7 at 1732521537036Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732521537036Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. at 1732521537036Flushing d7101967d1e1f9b0cf2c8406f99fcab7/testFamily: creating writer at 1732521537037 (+1 ms)Flushing d7101967d1e1f9b0cf2c8406f99fcab7/testFamily: appending metadata at 1732521537054 (+17 ms)Flushing d7101967d1e1f9b0cf2c8406f99fcab7/testFamily: closing flushed file at 1732521537054Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7633928d: reopening flushed file at 1732521537470 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d7101967d1e1f9b0cf2c8406f99fcab7 in 449ms, sequenceid=8, compaction requested=false at 1732521537484 (+14 ms)Writing region close event to WAL at 1732521537485 (+1 ms)Running coprocessor post-close hooks at 1732521537490 (+5 ms)Closed at 1732521537491 (+1 ms) 2024-11-25T07:58:57,491 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732521497919.d7101967d1e1f9b0cf2c8406f99fcab7. 2024-11-25T07:58:57,491 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b412ace892c85f02a9cc16735a0cce4c, disabling compactions & flushes 2024-11-25T07:58:57,491 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. 2024-11-25T07:58:57,491 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. 2024-11-25T07:58:57,491 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. after waiting 0 ms 2024-11-25T07:58:57,492 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. 2024-11-25T07:58:57,492 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b412ace892c85f02a9cc16735a0cce4c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T07:58:57,498 INFO [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/info/c39f9875041d4aa5a719b02366fbbc22, entries=100, sequenceid=56, filesize=20.1 K 2024-11-25T07:58:57,500 DEBUG [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/.tmp/ns/2ee25bd52c5645e0af690a49957f1a52 as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/ns/2ee25bd52c5645e0af690a49957f1a52 2024-11-25T07:58:57,510 INFO [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/ns/2ee25bd52c5645e0af690a49957f1a52, entries=2, sequenceid=56, filesize=5.0 K 2024-11-25T07:58:57,511 DEBUG [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/.tmp/table/de20097d1027433896f388f6ec3434f0 as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/table/de20097d1027433896f388f6ec3434f0 2024-11-25T07:58:57,514 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b412ace892c85f02a9cc16735a0cce4c/.tmp/testFamily/aaf3196f19294e02844144bf0b2d2052 is 61, key is row-0/testFamily:testQualifier/1732521517199/Put/seqid=0 2024-11-25T07:58:57,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741860_1036 (size=5049) 2024-11-25T07:58:57,520 INFO [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/table/de20097d1027433896f388f6ec3434f0, entries=20, sequenceid=56, filesize=6.5 K 2024-11-25T07:58:57,521 INFO [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1036ms, sequenceid=56, compaction requested=false 2024-11-25T07:58:57,526 DEBUG [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-25T07:58:57,526 DEBUG [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T07:58:57,527 INFO [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T07:58:57,527 DEBUG [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732521536485Running coprocessor pre-close hooks at 1732521536485Disabling compacts and flushes for region at 1732521536485Disabling writes for close at 1732521536485Obtaining lock to block concurrent updates at 1732521536488 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1732521536488Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732521536495 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732521536496 (+1 ms)Flushing 1588230740/info: creating writer at 1732521536497 (+1 ms)Flushing 1588230740/info: appending metadata at 1732521536547 (+50 ms)Flushing 1588230740/info: closing flushed file at 1732521536547Flushing 1588230740/ns: creating writer at 1732521537006 (+459 ms)Flushing 1588230740/ns: appending metadata at 1732521537028 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1732521537029 (+1 ms)Flushing 1588230740/table: creating writer at 1732521537451 (+422 ms)Flushing 1588230740/table: appending metadata at 1732521537472 (+21 ms)Flushing 1588230740/table: closing flushed file at 1732521537472Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bff29cc: reopening flushed file at 1732521537487 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b354257: reopening flushed file at 1732521537499 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ebfa133: reopening flushed file at 1732521537510 (+11 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1036ms, sequenceid=56, compaction requested=false at 1732521537521 (+11 ms)Writing region close event to WAL at 1732521537522 (+1 ms)Running coprocessor post-close hooks at 1732521537526 (+4 ms)Closed at 1732521537527 (+1 ms) 2024-11-25T07:58:57,527 DEBUG [RS_CLOSE_META-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T07:58:57,687 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 6e414485380d7d765d4f3e076fcdb9f1, 908344ba8601916f4c690fceca67003a, b412ace892c85f02a9cc16735a0cce4c, f1cec251a05357f75aa8201c9d6696a0, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:57,888 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 6e414485380d7d765d4f3e076fcdb9f1, 908344ba8601916f4c690fceca67003a, b412ace892c85f02a9cc16735a0cce4c, f1cec251a05357f75aa8201c9d6696a0, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:57,921 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b412ace892c85f02a9cc16735a0cce4c/.tmp/testFamily/aaf3196f19294e02844144bf0b2d2052 2024-11-25T07:58:57,964 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b412ace892c85f02a9cc16735a0cce4c/.tmp/testFamily/aaf3196f19294e02844144bf0b2d2052 as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b412ace892c85f02a9cc16735a0cce4c/testFamily/aaf3196f19294e02844144bf0b2d2052 2024-11-25T07:58:57,974 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b412ace892c85f02a9cc16735a0cce4c/testFamily/aaf3196f19294e02844144bf0b2d2052, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T07:58:57,975 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b412ace892c85f02a9cc16735a0cce4c in 483ms, sequenceid=8, compaction requested=false 2024-11-25T07:58:57,981 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b412ace892c85f02a9cc16735a0cce4c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T07:58:57,982 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. 2024-11-25T07:58:57,982 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b412ace892c85f02a9cc16735a0cce4c: Waiting for close lock at 1732521537491Running coprocessor pre-close hooks at 1732521537491Disabling compacts and flushes for region at 1732521537491Disabling writes for close at 1732521537491Obtaining lock to block concurrent updates at 1732521537492 (+1 ms)Preparing flush snapshotting stores in b412ace892c85f02a9cc16735a0cce4c at 1732521537492Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732521537492Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. at 1732521537493 (+1 ms)Flushing b412ace892c85f02a9cc16735a0cce4c/testFamily: creating writer at 1732521537493Flushing b412ace892c85f02a9cc16735a0cce4c/testFamily: appending metadata at 1732521537514 (+21 ms)Flushing b412ace892c85f02a9cc16735a0cce4c/testFamily: closing flushed file at 1732521537514Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b86ab24: reopening flushed file at 1732521537962 (+448 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b412ace892c85f02a9cc16735a0cce4c in 483ms, sequenceid=8, compaction requested=false at 1732521537975 (+13 ms)Writing region close event to WAL at 1732521537976 (+1 ms)Running coprocessor post-close hooks at 1732521537982 (+6 ms)Closed at 1732521537982 2024-11-25T07:58:57,983 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732521514151.b412ace892c85f02a9cc16735a0cce4c. 2024-11-25T07:58:57,983 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f1cec251a05357f75aa8201c9d6696a0, disabling compactions & flushes 2024-11-25T07:58:57,983 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. 2024-11-25T07:58:57,983 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. 2024-11-25T07:58:57,983 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. after waiting 0 ms 2024-11-25T07:58:57,983 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. 2024-11-25T07:58:57,983 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f1cec251a05357f75aa8201c9d6696a0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T07:58:58,001 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNextAsync/f1cec251a05357f75aa8201c9d6696a0/.tmp/testFamily/1ce9572a2b374c7fb0204a3ffa23f9fa is 61, key is row-0/testFamily:testQualifier/1732521505035/Put/seqid=0 2024-11-25T07:58:58,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741861_1037 (size=5049) 2024-11-25T07:58:58,088 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 6e414485380d7d765d4f3e076fcdb9f1, 908344ba8601916f4c690fceca67003a, f1cec251a05357f75aa8201c9d6696a0, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:58,288 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 6e414485380d7d765d4f3e076fcdb9f1, 908344ba8601916f4c690fceca67003a, f1cec251a05357f75aa8201c9d6696a0, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:58,408 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNextAsync/f1cec251a05357f75aa8201c9d6696a0/.tmp/testFamily/1ce9572a2b374c7fb0204a3ffa23f9fa 2024-11-25T07:58:58,422 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNextAsync/f1cec251a05357f75aa8201c9d6696a0/.tmp/testFamily/1ce9572a2b374c7fb0204a3ffa23f9fa as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNextAsync/f1cec251a05357f75aa8201c9d6696a0/testFamily/1ce9572a2b374c7fb0204a3ffa23f9fa 2024-11-25T07:58:58,430 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNextAsync/f1cec251a05357f75aa8201c9d6696a0/testFamily/1ce9572a2b374c7fb0204a3ffa23f9fa, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T07:58:58,431 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f1cec251a05357f75aa8201c9d6696a0 in 448ms, sequenceid=8, compaction requested=false 2024-11-25T07:58:58,436 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNextAsync/f1cec251a05357f75aa8201c9d6696a0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T07:58:58,437 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. 2024-11-25T07:58:58,437 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f1cec251a05357f75aa8201c9d6696a0: Waiting for close lock at 1732521537983Running coprocessor pre-close hooks at 1732521537983Disabling compacts and flushes for region at 1732521537983Disabling writes for close at 1732521537983Obtaining lock to block concurrent updates at 1732521537983Preparing flush snapshotting stores in f1cec251a05357f75aa8201c9d6696a0 at 1732521537983Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732521537983Flushing stores of testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. at 1732521537984 (+1 ms)Flushing f1cec251a05357f75aa8201c9d6696a0/testFamily: creating writer at 1732521537984Flushing f1cec251a05357f75aa8201c9d6696a0/testFamily: appending metadata at 1732521538001 (+17 ms)Flushing f1cec251a05357f75aa8201c9d6696a0/testFamily: closing flushed file at 1732521538001Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@515b8030: reopening flushed file at 1732521538421 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f1cec251a05357f75aa8201c9d6696a0 in 448ms, sequenceid=8, compaction requested=false at 1732521538431 (+10 ms)Writing region close event to WAL at 1732521538432 (+1 ms)Running coprocessor post-close hooks at 1732521538437 (+5 ms)Closed at 1732521538437 2024-11-25T07:58:58,438 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732521504008.f1cec251a05357f75aa8201c9d6696a0. 2024-11-25T07:58:58,438 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6e414485380d7d765d4f3e076fcdb9f1, disabling compactions & flushes 2024-11-25T07:58:58,438 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. 2024-11-25T07:58:58,438 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. 2024-11-25T07:58:58,438 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. after waiting 0 ms 2024-11-25T07:58:58,438 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. 2024-11-25T07:58:58,438 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6e414485380d7d765d4f3e076fcdb9f1 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T07:58:58,456 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNext/6e414485380d7d765d4f3e076fcdb9f1/.tmp/testFamily/c8136e9ecff34efd85ef319789d492f3 is 61, key is row-0/testFamily:testQualifier/1732521524274/Put/seqid=0 2024-11-25T07:58:58,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741862_1038 (size=5049) 2024-11-25T07:58:58,489 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-25T07:58:58,489 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1325): Online Regions={6e414485380d7d765d4f3e076fcdb9f1=hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1., 908344ba8601916f4c690fceca67003a=testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a., 470929e933e6a8a588c3fc1fd5ba5065=hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065., ffa3712a288e16bfac8069191f2eda48=hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48., 239239bec824bf759590c29a95c9a1b2=testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., 148d94d6f40d416655c1aa51d449d387=testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387.} 2024-11-25T07:58:58,489 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 6e414485380d7d765d4f3e076fcdb9f1, 908344ba8601916f4c690fceca67003a, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:58,649 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-25T07:58:58,649 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T07:58:58,651 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T07:58:58,651 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-25T07:58:58,652 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T07:58:58,652 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-25T07:58:58,652 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-25T07:58:58,652 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-25T07:58:58,689 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 6e414485380d7d765d4f3e076fcdb9f1, 908344ba8601916f4c690fceca67003a, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:58,863 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNext/6e414485380d7d765d4f3e076fcdb9f1/.tmp/testFamily/c8136e9ecff34efd85ef319789d492f3 2024-11-25T07:58:58,880 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNext/6e414485380d7d765d4f3e076fcdb9f1/.tmp/testFamily/c8136e9ecff34efd85ef319789d492f3 as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNext/6e414485380d7d765d4f3e076fcdb9f1/testFamily/c8136e9ecff34efd85ef319789d492f3 2024-11-25T07:58:58,890 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 6e414485380d7d765d4f3e076fcdb9f1, 908344ba8601916f4c690fceca67003a, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:58,890 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNext/6e414485380d7d765d4f3e076fcdb9f1/testFamily/c8136e9ecff34efd85ef319789d492f3, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T07:58:58,891 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6e414485380d7d765d4f3e076fcdb9f1 in 453ms, sequenceid=8, compaction requested=false 2024-11-25T07:58:58,896 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNext/6e414485380d7d765d4f3e076fcdb9f1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T07:58:58,897 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. 2024-11-25T07:58:58,897 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6e414485380d7d765d4f3e076fcdb9f1: Waiting for close lock at 1732521538438Running coprocessor pre-close hooks at 1732521538438Disabling compacts and flushes for region at 1732521538438Disabling writes for close at 1732521538438Obtaining lock to block concurrent updates at 1732521538438Preparing flush snapshotting stores in 6e414485380d7d765d4f3e076fcdb9f1 at 1732521538438Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732521538438Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. at 1732521538439 (+1 ms)Flushing 6e414485380d7d765d4f3e076fcdb9f1/testFamily: creating writer at 1732521538439Flushing 6e414485380d7d765d4f3e076fcdb9f1/testFamily: appending metadata at 1732521538456 (+17 ms)Flushing 6e414485380d7d765d4f3e076fcdb9f1/testFamily: closing flushed file at 1732521538456Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62bbea38: reopening flushed file at 1732521538878 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6e414485380d7d765d4f3e076fcdb9f1 in 453ms, sequenceid=8, compaction requested=false at 1732521538891 (+13 ms)Writing region close event to WAL at 1732521538892 (+1 ms)Running coprocessor post-close hooks at 1732521538896 (+4 ms)Closed at 1732521538896 2024-11-25T07:58:58,897 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732521523241.6e414485380d7d765d4f3e076fcdb9f1. 2024-11-25T07:58:58,897 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 908344ba8601916f4c690fceca67003a, disabling compactions & flushes 2024-11-25T07:58:58,897 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. 2024-11-25T07:58:58,897 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. 2024-11-25T07:58:58,897 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. after waiting 0 ms 2024-11-25T07:58:58,897 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. 2024-11-25T07:58:58,897 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 908344ba8601916f4c690fceca67003a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T07:58:58,913 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNext/908344ba8601916f4c690fceca67003a/.tmp/testFamily/fa81603d664442a782917ed49c7fd168 is 61, key is row-0/testFamily:testQualifier/1732521511118/Put/seqid=0 2024-11-25T07:58:58,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741863_1039 (size=5049) 2024-11-25T07:58:59,090 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 908344ba8601916f4c690fceca67003a, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:59,290 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, 908344ba8601916f4c690fceca67003a, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:59,319 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNext/908344ba8601916f4c690fceca67003a/.tmp/testFamily/fa81603d664442a782917ed49c7fd168 2024-11-25T07:58:59,335 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNext/908344ba8601916f4c690fceca67003a/.tmp/testFamily/fa81603d664442a782917ed49c7fd168 as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNext/908344ba8601916f4c690fceca67003a/testFamily/fa81603d664442a782917ed49c7fd168 2024-11-25T07:58:59,344 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNext/908344ba8601916f4c690fceca67003a/testFamily/fa81603d664442a782917ed49c7fd168, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T07:58:59,345 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 908344ba8601916f4c690fceca67003a in 448ms, sequenceid=8, compaction requested=false 2024-11-25T07:58:59,350 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnNext/908344ba8601916f4c690fceca67003a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T07:58:59,351 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. 2024-11-25T07:58:59,351 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 908344ba8601916f4c690fceca67003a: Waiting for close lock at 1732521538897Running coprocessor pre-close hooks at 1732521538897Disabling compacts and flushes for region at 1732521538897Disabling writes for close at 1732521538897Obtaining lock to block concurrent updates at 1732521538897Preparing flush snapshotting stores in 908344ba8601916f4c690fceca67003a at 1732521538897Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732521538898 (+1 ms)Flushing stores of testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. at 1732521538898Flushing 908344ba8601916f4c690fceca67003a/testFamily: creating writer at 1732521538898Flushing 908344ba8601916f4c690fceca67003a/testFamily: appending metadata at 1732521538913 (+15 ms)Flushing 908344ba8601916f4c690fceca67003a/testFamily: closing flushed file at 1732521538913Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d0c2fe2: reopening flushed file at 1732521539332 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 908344ba8601916f4c690fceca67003a in 448ms, sequenceid=8, compaction requested=false at 1732521539345 (+13 ms)Writing region close event to WAL at 1732521539346 (+1 ms)Running coprocessor post-close hooks at 1732521539351 (+5 ms)Closed at 1732521539351 2024-11-25T07:58:59,351 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732521508077.908344ba8601916f4c690fceca67003a. 2024-11-25T07:58:59,351 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 470929e933e6a8a588c3fc1fd5ba5065, disabling compactions & flushes 2024-11-25T07:58:59,351 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. 2024-11-25T07:58:59,351 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. 2024-11-25T07:58:59,352 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. after waiting 0 ms 2024-11-25T07:58:59,352 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. 2024-11-25T07:58:59,352 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 470929e933e6a8a588c3fc1fd5ba5065 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T07:58:59,369 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNextAsync/470929e933e6a8a588c3fc1fd5ba5065/.tmp/testFamily/dc10a28ffd62425599ed24419f5585b6 is 61, key is row-0/testFamily:testQualifier/1732521488759/Put/seqid=0 2024-11-25T07:58:59,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741864_1040 (size=5049) 2024-11-25T07:58:59,491 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-25T07:58:59,491 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1325): Online Regions={470929e933e6a8a588c3fc1fd5ba5065=hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065., ffa3712a288e16bfac8069191f2eda48=hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48., 239239bec824bf759590c29a95c9a1b2=testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., 148d94d6f40d416655c1aa51d449d387=testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387.} 2024-11-25T07:58:59,491 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:59,691 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2, 470929e933e6a8a588c3fc1fd5ba5065, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:58:59,776 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNextAsync/470929e933e6a8a588c3fc1fd5ba5065/.tmp/testFamily/dc10a28ffd62425599ed24419f5585b6 2024-11-25T07:58:59,783 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNextAsync/470929e933e6a8a588c3fc1fd5ba5065/.tmp/testFamily/dc10a28ffd62425599ed24419f5585b6 as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNextAsync/470929e933e6a8a588c3fc1fd5ba5065/testFamily/dc10a28ffd62425599ed24419f5585b6 2024-11-25T07:58:59,791 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNextAsync/470929e933e6a8a588c3fc1fd5ba5065/testFamily/dc10a28ffd62425599ed24419f5585b6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T07:58:59,792 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 470929e933e6a8a588c3fc1fd5ba5065 in 440ms, sequenceid=8, compaction requested=false 2024-11-25T07:58:59,797 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnNextAsync/470929e933e6a8a588c3fc1fd5ba5065/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T07:58:59,798 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. 2024-11-25T07:58:59,798 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 470929e933e6a8a588c3fc1fd5ba5065: Waiting for close lock at 1732521539351Running coprocessor pre-close hooks at 1732521539351Disabling compacts and flushes for region at 1732521539351Disabling writes for close at 1732521539352 (+1 ms)Obtaining lock to block concurrent updates at 1732521539352Preparing flush snapshotting stores in 470929e933e6a8a588c3fc1fd5ba5065 at 1732521539352Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732521539352Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. at 1732521539353 (+1 ms)Flushing 470929e933e6a8a588c3fc1fd5ba5065/testFamily: creating writer at 1732521539353Flushing 470929e933e6a8a588c3fc1fd5ba5065/testFamily: appending metadata at 1732521539369 (+16 ms)Flushing 470929e933e6a8a588c3fc1fd5ba5065/testFamily: closing flushed file at 1732521539369Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38fbd9f4: reopening flushed file at 1732521539782 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 470929e933e6a8a588c3fc1fd5ba5065 in 440ms, sequenceid=8, compaction requested=false at 1732521539792 (+10 ms)Writing region close event to WAL at 1732521539792Running coprocessor post-close hooks at 1732521539798 (+6 ms)Closed at 1732521539798 2024-11-25T07:58:59,798 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732521485716.470929e933e6a8a588c3fc1fd5ba5065. 2024-11-25T07:58:59,798 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ffa3712a288e16bfac8069191f2eda48, disabling compactions & flushes 2024-11-25T07:58:59,798 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. 2024-11-25T07:58:59,798 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. 2024-11-25T07:58:59,798 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. after waiting 0 ms 2024-11-25T07:58:59,798 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. 2024-11-25T07:58:59,798 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ffa3712a288e16bfac8069191f2eda48 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T07:58:59,815 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScanner/ffa3712a288e16bfac8069191f2eda48/.tmp/testFamily/11b61048035649b88d993092025affc2 is 61, key is row-0/testFamily:testQualifier/1732521479658/Put/seqid=0 2024-11-25T07:58:59,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741865_1041 (size=5049) 2024-11-25T07:58:59,892 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:59:00,092 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2, ffa3712a288e16bfac8069191f2eda48 2024-11-25T07:59:00,175 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=7fb8763a5d32:35737 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 10 more 2024-11-25T07:59:00,220 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScanner/ffa3712a288e16bfac8069191f2eda48/.tmp/testFamily/11b61048035649b88d993092025affc2 2024-11-25T07:59:00,233 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScanner/ffa3712a288e16bfac8069191f2eda48/.tmp/testFamily/11b61048035649b88d993092025affc2 as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScanner/ffa3712a288e16bfac8069191f2eda48/testFamily/11b61048035649b88d993092025affc2 2024-11-25T07:59:00,241 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScanner/ffa3712a288e16bfac8069191f2eda48/testFamily/11b61048035649b88d993092025affc2, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T07:59:00,242 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ffa3712a288e16bfac8069191f2eda48 in 444ms, sequenceid=8, compaction requested=false 2024-11-25T07:59:00,247 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/hbase/testMetaScanTimeoutOnOpenScanner/ffa3712a288e16bfac8069191f2eda48/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T07:59:00,247 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. 2024-11-25T07:59:00,247 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ffa3712a288e16bfac8069191f2eda48: Waiting for close lock at 1732521539798Running coprocessor pre-close hooks at 1732521539798Disabling compacts and flushes for region at 1732521539798Disabling writes for close at 1732521539798Obtaining lock to block concurrent updates at 1732521539798Preparing flush snapshotting stores in ffa3712a288e16bfac8069191f2eda48 at 1732521539798Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732521539799 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. at 1732521539799Flushing ffa3712a288e16bfac8069191f2eda48/testFamily: creating writer at 1732521539799Flushing ffa3712a288e16bfac8069191f2eda48/testFamily: appending metadata at 1732521539815 (+16 ms)Flushing ffa3712a288e16bfac8069191f2eda48/testFamily: closing flushed file at 1732521539815Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6179dafe: reopening flushed file at 1732521540232 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ffa3712a288e16bfac8069191f2eda48 in 444ms, sequenceid=8, compaction requested=false at 1732521540242 (+10 ms)Writing region close event to WAL at 1732521540243 (+1 ms)Running coprocessor post-close hooks at 1732521540247 (+4 ms)Closed at 1732521540247 2024-11-25T07:59:00,247 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732521476595.ffa3712a288e16bfac8069191f2eda48. 2024-11-25T07:59:00,248 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 239239bec824bf759590c29a95c9a1b2, disabling compactions & flushes 2024-11-25T07:59:00,248 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. 2024-11-25T07:59:00,248 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. 2024-11-25T07:59:00,248 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. after waiting 0 ms 2024-11-25T07:59:00,248 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. 2024-11-25T07:59:00,248 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 239239bec824bf759590c29a95c9a1b2 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T07:59:00,264 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScannerAsync/239239bec824bf759590c29a95c9a1b2/.tmp/testFamily/2d97e642907c43dca26696b51f8fcce3 is 61, key is row-0/testFamily:testQualifier/1732521533374/Put/seqid=0 2024-11-25T07:59:00,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741866_1042 (size=5049) 2024-11-25T07:59:00,292 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2 2024-11-25T07:59:00,492 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T07:59:00,493 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1325): Online Regions={239239bec824bf759590c29a95c9a1b2=testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., 148d94d6f40d416655c1aa51d449d387=testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387.} 2024-11-25T07:59:00,493 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2 2024-11-25T07:59:00,670 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScannerAsync/239239bec824bf759590c29a95c9a1b2/.tmp/testFamily/2d97e642907c43dca26696b51f8fcce3 2024-11-25T07:59:00,685 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScannerAsync/239239bec824bf759590c29a95c9a1b2/.tmp/testFamily/2d97e642907c43dca26696b51f8fcce3 as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScannerAsync/239239bec824bf759590c29a95c9a1b2/testFamily/2d97e642907c43dca26696b51f8fcce3 2024-11-25T07:59:00,693 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387, 239239bec824bf759590c29a95c9a1b2 2024-11-25T07:59:00,693 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScannerAsync/239239bec824bf759590c29a95c9a1b2/testFamily/2d97e642907c43dca26696b51f8fcce3, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T07:59:00,694 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 239239bec824bf759590c29a95c9a1b2 in 446ms, sequenceid=8, compaction requested=false 2024-11-25T07:59:00,699 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testNormalScanTimeoutOnOpenScannerAsync/239239bec824bf759590c29a95c9a1b2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T07:59:00,700 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. 2024-11-25T07:59:00,700 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 239239bec824bf759590c29a95c9a1b2: Waiting for close lock at 1732521540248Running coprocessor pre-close hooks at 1732521540248Disabling compacts and flushes for region at 1732521540248Disabling writes for close at 1732521540248Obtaining lock to block concurrent updates at 1732521540248Preparing flush snapshotting stores in 239239bec824bf759590c29a95c9a1b2 at 1732521540248Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732521540248Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. at 1732521540249 (+1 ms)Flushing 239239bec824bf759590c29a95c9a1b2/testFamily: creating writer at 1732521540249Flushing 239239bec824bf759590c29a95c9a1b2/testFamily: appending metadata at 1732521540263 (+14 ms)Flushing 239239bec824bf759590c29a95c9a1b2/testFamily: closing flushed file at 1732521540263Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18a8b645: reopening flushed file at 1732521540684 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 239239bec824bf759590c29a95c9a1b2 in 446ms, sequenceid=8, compaction requested=false at 1732521540694 (+10 ms)Writing region close event to WAL at 1732521540695 (+1 ms)Running coprocessor post-close hooks at 1732521540700 (+5 ms)Closed at 1732521540700 2024-11-25T07:59:00,700 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732521530327.239239bec824bf759590c29a95c9a1b2. 2024-11-25T07:59:00,700 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 148d94d6f40d416655c1aa51d449d387, disabling compactions & flushes 2024-11-25T07:59:00,700 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. 2024-11-25T07:59:00,700 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. 2024-11-25T07:59:00,700 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. after waiting 0 ms 2024-11-25T07:59:00,700 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. 2024-11-25T07:59:00,700 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 148d94d6f40d416655c1aa51d449d387 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T07:59:00,714 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextException/148d94d6f40d416655c1aa51d449d387/.tmp/testFamily/4e6d6fa5f19c4d149167143ebdbae1cc is 61, key is row-0/testFamily:testQualifier/1732521497868/Put/seqid=0 2024-11-25T07:59:00,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741867_1043 (size=5049) 2024-11-25T07:59:00,893 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387 2024-11-25T07:59:01,094 DEBUG [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1351): Waiting on 148d94d6f40d416655c1aa51d449d387 2024-11-25T07:59:01,120 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextException/148d94d6f40d416655c1aa51d449d387/.tmp/testFamily/4e6d6fa5f19c4d149167143ebdbae1cc 2024-11-25T07:59:01,131 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextException/148d94d6f40d416655c1aa51d449d387/.tmp/testFamily/4e6d6fa5f19c4d149167143ebdbae1cc as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextException/148d94d6f40d416655c1aa51d449d387/testFamily/4e6d6fa5f19c4d149167143ebdbae1cc 2024-11-25T07:59:01,140 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextException/148d94d6f40d416655c1aa51d449d387/testFamily/4e6d6fa5f19c4d149167143ebdbae1cc, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T07:59:01,141 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 148d94d6f40d416655c1aa51d449d387 in 441ms, sequenceid=8, compaction requested=false 2024-11-25T07:59:01,146 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/data/default/testRetryOutOfOrderScannerNextException/148d94d6f40d416655c1aa51d449d387/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T07:59:01,147 INFO [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. 2024-11-25T07:59:01,147 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 148d94d6f40d416655c1aa51d449d387: Waiting for close lock at 1732521540700Running coprocessor pre-close hooks at 1732521540700Disabling compacts and flushes for region at 1732521540700Disabling writes for close at 1732521540700Obtaining lock to block concurrent updates at 1732521540700Preparing flush snapshotting stores in 148d94d6f40d416655c1aa51d449d387 at 1732521540700Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732521540700Flushing stores of testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. at 1732521540701 (+1 ms)Flushing 148d94d6f40d416655c1aa51d449d387/testFamily: creating writer at 1732521540701Flushing 148d94d6f40d416655c1aa51d449d387/testFamily: appending metadata at 1732521540714 (+13 ms)Flushing 148d94d6f40d416655c1aa51d449d387/testFamily: closing flushed file at 1732521540714Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@522b92de: reopening flushed file at 1732521541129 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 148d94d6f40d416655c1aa51d449d387 in 441ms, sequenceid=8, compaction requested=false at 1732521541141 (+12 ms)Writing region close event to WAL at 1732521541142 (+1 ms)Running coprocessor post-close hooks at 1732521541147 (+5 ms)Closed at 1732521541147 2024-11-25T07:59:01,148 DEBUG [RS_CLOSE_REGION-regionserver/7fb8763a5d32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732521494806.148d94d6f40d416655c1aa51d449d387. 2024-11-25T07:59:01,294 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(976): stopping server 7fb8763a5d32,41809,1732521439188; all regions closed. 2024-11-25T07:59:01,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741834_1010 (size=19479) 2024-11-25T07:59:01,306 DEBUG [RS:0;7fb8763a5d32:41809 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/oldWALs 2024-11-25T07:59:01,306 INFO [RS:0;7fb8763a5d32:41809 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7fb8763a5d32%2C41809%2C1732521439188.meta:.meta(num 1732521472798) 2024-11-25T07:59:01,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741833_1009 (size=20031) 2024-11-25T07:59:01,309 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/WALs/7fb8763a5d32,41809,1732521439188/7fb8763a5d32%2C41809%2C1732521439188.1732521472412 not finished, retry = 0 2024-11-25T07:59:01,413 DEBUG [RS:0;7fb8763a5d32:41809 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/oldWALs 2024-11-25T07:59:01,413 INFO [RS:0;7fb8763a5d32:41809 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7fb8763a5d32%2C41809%2C1732521439188:(num 1732521472412) 2024-11-25T07:59:01,413 DEBUG [RS:0;7fb8763a5d32:41809 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T07:59:01,413 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T07:59:01,413 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T07:59:01,413 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.ChoreService(370): Chore service for: regionserver/7fb8763a5d32:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T07:59:01,413 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T07:59:01,413 INFO [regionserver/7fb8763a5d32:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T07:59:01,415 INFO [RS:0;7fb8763a5d32:41809 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41809 2024-11-25T07:59:01,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T07:59:01,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7fb8763a5d32,41809,1732521439188 2024-11-25T07:59:01,433 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T07:59:01,442 INFO [regionserver/7fb8763a5d32:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T07:59:01,455 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7fb8763a5d32,41809,1732521439188] 2024-11-25T07:59:01,465 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7fb8763a5d32,41809,1732521439188 already deleted, retry=false 2024-11-25T07:59:01,465 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7fb8763a5d32,41809,1732521439188 expired; onlineServers=0 2024-11-25T07:59:01,465 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7fb8763a5d32,35737,1732521438242' ***** 2024-11-25T07:59:01,465 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T07:59:01,465 INFO [M:0;7fb8763a5d32:35737 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T07:59:01,465 INFO [M:0;7fb8763a5d32:35737 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T07:59:01,466 DEBUG [M:0;7fb8763a5d32:35737 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T07:59:01,466 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T07:59:01,466 DEBUG [M:0;7fb8763a5d32:35737 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T07:59:01,466 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster-HFileCleaner.large.0-1732521442206 {}] cleaner.HFileCleaner(306): Exit Thread[master/7fb8763a5d32:0:becomeActiveMaster-HFileCleaner.large.0-1732521442206,5,FailOnTimeoutGroup] 2024-11-25T07:59:01,466 DEBUG [master/7fb8763a5d32:0:becomeActiveMaster-HFileCleaner.small.0-1732521442207 {}] cleaner.HFileCleaner(306): Exit Thread[master/7fb8763a5d32:0:becomeActiveMaster-HFileCleaner.small.0-1732521442207,5,FailOnTimeoutGroup] 2024-11-25T07:59:01,467 INFO [M:0;7fb8763a5d32:35737 {}] hbase.ChoreService(370): Chore service for: master/7fb8763a5d32:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T07:59:01,467 INFO [M:0;7fb8763a5d32:35737 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T07:59:01,467 DEBUG [M:0;7fb8763a5d32:35737 {}] master.HMaster(1795): Stopping service threads 2024-11-25T07:59:01,467 INFO [M:0;7fb8763a5d32:35737 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T07:59:01,467 INFO [M:0;7fb8763a5d32:35737 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T07:59:01,468 INFO [M:0;7fb8763a5d32:35737 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T07:59:01,468 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T07:59:01,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T07:59:01,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T07:59:01,476 DEBUG [M:0;7fb8763a5d32:35737 {}] zookeeper.ZKUtil(347): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T07:59:01,476 WARN [M:0;7fb8763a5d32:35737 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T07:59:01,477 INFO [M:0;7fb8763a5d32:35737 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/.lastflushedseqids 2024-11-25T07:59:01,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741868_1044 (size=721) 2024-11-25T07:59:01,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T07:59:01,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41809-0x10170ff24600001, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T07:59:01,556 INFO [RS:0;7fb8763a5d32:41809 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T07:59:01,556 INFO [RS:0;7fb8763a5d32:41809 {}] regionserver.HRegionServer(1031): Exiting; stopping=7fb8763a5d32,41809,1732521439188; zookeeper connection closed. 2024-11-25T07:59:01,556 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3025d7f8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3025d7f8 2024-11-25T07:59:01,556 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-25T07:59:01,891 INFO [M:0;7fb8763a5d32:35737 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T07:59:01,891 INFO [M:0;7fb8763a5d32:35737 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T07:59:01,891 DEBUG [M:0;7fb8763a5d32:35737 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T07:59:01,891 INFO [M:0;7fb8763a5d32:35737 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:59:01,891 DEBUG [M:0;7fb8763a5d32:35737 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:59:01,891 DEBUG [M:0;7fb8763a5d32:35737 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T07:59:01,891 DEBUG [M:0;7fb8763a5d32:35737 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:59:01,892 INFO [M:0;7fb8763a5d32:35737 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.03 KB heapSize=191.06 KB 2024-11-25T07:59:01,910 DEBUG [M:0;7fb8763a5d32:35737 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/63904299420247a3807243df57b145ff is 82, key is hbase:meta,,1/info:regioninfo/1732521472898/Put/seqid=0 2024-11-25T07:59:01,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741869_1045 (size=5672) 2024-11-25T07:59:02,317 INFO [M:0;7fb8763a5d32:35737 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/63904299420247a3807243df57b145ff 2024-11-25T07:59:02,351 DEBUG [M:0;7fb8763a5d32:35737 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/49ae7957970b48fb92c2bcedff5ebade is 811, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732521474680/Put/seqid=0 2024-11-25T07:59:02,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741870_1046 (size=14066) 2024-11-25T07:59:02,536 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T07:59:02,768 INFO [M:0;7fb8763a5d32:35737 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.47 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/49ae7957970b48fb92c2bcedff5ebade 2024-11-25T07:59:02,776 INFO [M:0;7fb8763a5d32:35737 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 49ae7957970b48fb92c2bcedff5ebade 2024-11-25T07:59:02,792 DEBUG [M:0;7fb8763a5d32:35737 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/854d89dc8ff34ec5b9cd6a953e441f6b is 69, key is 7fb8763a5d32,41809,1732521439188/rs:state/1732521442251/Put/seqid=0 2024-11-25T07:59:02,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741871_1047 (size=5156) 2024-11-25T07:59:03,200 INFO [M:0;7fb8763a5d32:35737 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/854d89dc8ff34ec5b9cd6a953e441f6b 2024-11-25T07:59:03,212 DEBUG [M:0;7fb8763a5d32:35737 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/63904299420247a3807243df57b145ff as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/63904299420247a3807243df57b145ff 2024-11-25T07:59:03,218 INFO [M:0;7fb8763a5d32:35737 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/63904299420247a3807243df57b145ff, entries=8, sequenceid=319, filesize=5.5 K 2024-11-25T07:59:03,219 DEBUG [M:0;7fb8763a5d32:35737 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/49ae7957970b48fb92c2bcedff5ebade as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/49ae7957970b48fb92c2bcedff5ebade 2024-11-25T07:59:03,225 INFO [M:0;7fb8763a5d32:35737 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 49ae7957970b48fb92c2bcedff5ebade 2024-11-25T07:59:03,226 INFO [M:0;7fb8763a5d32:35737 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/49ae7957970b48fb92c2bcedff5ebade, entries=33, sequenceid=319, filesize=13.7 K 2024-11-25T07:59:03,227 DEBUG [M:0;7fb8763a5d32:35737 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/854d89dc8ff34ec5b9cd6a953e441f6b as hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/854d89dc8ff34ec5b9cd6a953e441f6b 2024-11-25T07:59:03,233 INFO [M:0;7fb8763a5d32:35737 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37877/user/jenkins/test-data/a4c764ee-1bb2-2588-bd23-168428fce574/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/854d89dc8ff34ec5b9cd6a953e441f6b, entries=1, sequenceid=319, filesize=5.0 K 2024-11-25T07:59:03,235 INFO [M:0;7fb8763a5d32:35737 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.03 KB/166943, heapSize ~190.77 KB/195344, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1342ms, sequenceid=319, compaction requested=false 2024-11-25T07:59:03,236 INFO [M:0;7fb8763a5d32:35737 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T07:59:03,236 DEBUG [M:0;7fb8763a5d32:35737 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732521541891Disabling compacts and flushes for region at 1732521541891Disabling writes for close at 1732521541891Obtaining lock to block concurrent updates at 1732521541892 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732521541892Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166943, getHeapSize=195584, getOffHeapSize=0, getCellsCount=366 at 1732521541892Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732521541893 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732521541893Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732521541909 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732521541909Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732521542332 (+423 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732521542350 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732521542350Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732521542776 (+426 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732521542791 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732521542791Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2247ce5: reopening flushed file at 1732521543210 (+419 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@520767b8: reopening flushed file at 1732521543218 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a29b75f: reopening flushed file at 1732521543226 (+8 ms)Finished flush of dataSize ~163.03 KB/166943, heapSize ~190.77 KB/195344, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1342ms, sequenceid=319, compaction requested=false at 1732521543235 (+9 ms)Writing region close event to WAL at 1732521543236 (+1 ms)Closed at 1732521543236 2024-11-25T07:59:03,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741830_1006 (size=188285) 2024-11-25T07:59:03,241 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T07:59:03,241 INFO [M:0;7fb8763a5d32:35737 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T07:59:03,241 INFO [M:0;7fb8763a5d32:35737 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35737 2024-11-25T07:59:03,241 INFO [M:0;7fb8763a5d32:35737 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T07:59:03,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T07:59:03,386 INFO [M:0;7fb8763a5d32:35737 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T07:59:03,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35737-0x10170ff24600000, quorum=127.0.0.1:53620, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T07:59:03,393 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@429b6cf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T07:59:03,396 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T07:59:03,396 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T07:59:03,396 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T07:59:03,396 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/hadoop.log.dir/,STOPPED} 2024-11-25T07:59:03,398 WARN [BP-1047878032-172.17.0.2-1732521433990 heartbeating to localhost/127.0.0.1:37877 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T07:59:03,398 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T07:59:03,399 WARN [BP-1047878032-172.17.0.2-1732521433990 heartbeating to localhost/127.0.0.1:37877 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1047878032-172.17.0.2-1732521433990 (Datanode Uuid b3316b2a-35b9-482a-9f92-d7e9b3114fac) service to localhost/127.0.0.1:37877 2024-11-25T07:59:03,399 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T07:59:03,400 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/cluster_ae7675c4-28d6-36d0-1e5b-73429a44c766/data/data1/current/BP-1047878032-172.17.0.2-1732521433990 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T07:59:03,400 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/cluster_ae7675c4-28d6-36d0-1e5b-73429a44c766/data/data2/current/BP-1047878032-172.17.0.2-1732521433990 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T07:59:03,400 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T07:59:03,408 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@40b93e99{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T07:59:03,408 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T07:59:03,408 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T07:59:03,408 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T07:59:03,409 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b67f4d5-0f5d-d8db-9679-69829c0c7a3e/hadoop.log.dir/,STOPPED} 2024-11-25T07:59:03,416 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T07:59:03,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down