2024-11-27 07:36:35,606 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-27 07:36:35,626 main DEBUG Took 0.017883 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 07:36:35,626 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 07:36:35,627 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 07:36:35,628 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 07:36:35,629 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:36:35,637 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 07:36:35,650 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:36:35,652 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:36:35,653 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:36:35,653 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:36:35,654 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:36:35,654 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:36:35,656 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:36:35,656 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:36:35,657 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:36:35,657 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:36:35,659 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:36:35,659 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:36:35,660 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:36:35,660 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:36:35,661 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:36:35,662 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:36:35,662 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:36:35,663 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:36:35,663 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:36:35,664 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:36:35,665 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:36:35,665 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:36:35,666 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:36:35,666 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:36:35,667 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:36:35,667 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 07:36:35,669 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:36:35,671 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 07:36:35,674 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 07:36:35,675 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 07:36:35,677 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 07:36:35,678 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 07:36:35,689 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 07:36:35,693 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 07:36:35,695 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 07:36:35,696 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 07:36:35,697 main DEBUG createAppenders(={Console}) 2024-11-27 07:36:35,698 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-27 07:36:35,698 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-27 07:36:35,699 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-27 07:36:35,700 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 07:36:35,700 main DEBUG OutputStream closed 2024-11-27 07:36:35,701 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 07:36:35,701 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 07:36:35,702 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-27 07:36:35,804 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 07:36:35,808 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 07:36:35,810 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 07:36:35,812 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 07:36:35,813 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 07:36:35,813 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 07:36:35,814 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 07:36:35,815 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 07:36:35,815 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 07:36:35,816 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 07:36:35,817 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 07:36:35,817 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 07:36:35,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 07:36:35,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 07:36:35,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 07:36:35,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 07:36:35,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 07:36:35,820 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 07:36:35,823 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 07:36:35,823 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-27 07:36:35,823 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 07:36:35,824 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-27T07:36:35,842 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-27 07:36:35,846 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 07:36:35,847 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T07:36:36,107 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd 2024-11-27T07:36:36,128 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T07:36:36,150 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/cluster_2a34800c-648a-d9f7-feb3-2a3237ffb0ee, deleteOnExit=true 2024-11-27T07:36:36,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T07:36:36,152 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/test.cache.data in system properties and HBase conf 2024-11-27T07:36:36,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T07:36:36,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/hadoop.log.dir in system properties and HBase conf 2024-11-27T07:36:36,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T07:36:36,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T07:36:36,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T07:36:36,242 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T07:36:36,328 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T07:36:36,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T07:36:36,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T07:36:36,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T07:36:36,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T07:36:36,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T07:36:36,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T07:36:36,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T07:36:36,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T07:36:36,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T07:36:36,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/nfs.dump.dir in system properties and HBase conf 2024-11-27T07:36:36,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/java.io.tmpdir in system properties and HBase conf 2024-11-27T07:36:36,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T07:36:36,337 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T07:36:36,337 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T07:36:37,593 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T07:36:37,660 INFO [Time-limited test {}] log.Log(170): Logging initialized @2760ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T07:36:37,725 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T07:36:37,782 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T07:36:37,801 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T07:36:37,802 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T07:36:37,803 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T07:36:37,814 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T07:36:37,817 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3653a146{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/hadoop.log.dir/,AVAILABLE} 2024-11-27T07:36:37,818 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c1e5f71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T07:36:37,993 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4df87c21{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/java.io.tmpdir/jetty-localhost-37723-hadoop-hdfs-3_4_1-tests_jar-_-any-14197748178363107748/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T07:36:37,999 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@29047fa7{HTTP/1.1, (http/1.1)}{localhost:37723} 2024-11-27T07:36:38,000 INFO [Time-limited test {}] server.Server(415): Started @3101ms 2024-11-27T07:36:38,562 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T07:36:38,569 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T07:36:38,570 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T07:36:38,570 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T07:36:38,571 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T07:36:38,571 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29ee0fe9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/hadoop.log.dir/,AVAILABLE} 2024-11-27T07:36:38,572 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48199fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T07:36:38,672 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6261b108{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/java.io.tmpdir/jetty-localhost-40095-hadoop-hdfs-3_4_1-tests_jar-_-any-15965270793545542361/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T07:36:38,672 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@379257e4{HTTP/1.1, (http/1.1)}{localhost:40095} 2024-11-27T07:36:38,673 INFO [Time-limited test {}] server.Server(415): Started @3774ms 2024-11-27T07:36:38,733 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T07:36:40,217 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/cluster_2a34800c-648a-d9f7-feb3-2a3237ffb0ee/data/data2/current/BP-1981243590-172.17.0.3-1732692997009/current, will proceed with Du for space computation calculation, 2024-11-27T07:36:40,217 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/cluster_2a34800c-648a-d9f7-feb3-2a3237ffb0ee/data/data1/current/BP-1981243590-172.17.0.3-1732692997009/current, will proceed with Du for space computation calculation, 2024-11-27T07:36:40,244 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T07:36:40,299 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1dbcb7c5cc13fe2a with lease ID 0xc6b2ea7a81acfd8a: Processing first storage report for DS-82256989-cc76-4a34-88fc-db2c26fc19a1 from datanode DatanodeRegistration(127.0.0.1:46455, datanodeUuid=ec107a75-18f9-4cbf-9000-da7b0d22afbb, infoPort=40819, infoSecurePort=0, ipcPort=36841, storageInfo=lv=-57;cid=testClusterID;nsid=759010943;c=1732692997009) 2024-11-27T07:36:40,300 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1dbcb7c5cc13fe2a with lease ID 0xc6b2ea7a81acfd8a: from storage DS-82256989-cc76-4a34-88fc-db2c26fc19a1 node DatanodeRegistration(127.0.0.1:46455, datanodeUuid=ec107a75-18f9-4cbf-9000-da7b0d22afbb, infoPort=40819, infoSecurePort=0, ipcPort=36841, storageInfo=lv=-57;cid=testClusterID;nsid=759010943;c=1732692997009), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T07:36:40,301 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1dbcb7c5cc13fe2a with lease ID 0xc6b2ea7a81acfd8a: Processing first storage report for DS-31aa0bc7-4c19-48f3-9390-8fe8f4d1598e from datanode DatanodeRegistration(127.0.0.1:46455, datanodeUuid=ec107a75-18f9-4cbf-9000-da7b0d22afbb, infoPort=40819, infoSecurePort=0, ipcPort=36841, storageInfo=lv=-57;cid=testClusterID;nsid=759010943;c=1732692997009) 2024-11-27T07:36:40,301 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1dbcb7c5cc13fe2a with lease ID 0xc6b2ea7a81acfd8a: from storage DS-31aa0bc7-4c19-48f3-9390-8fe8f4d1598e node DatanodeRegistration(127.0.0.1:46455, datanodeUuid=ec107a75-18f9-4cbf-9000-da7b0d22afbb, infoPort=40819, infoSecurePort=0, ipcPort=36841, storageInfo=lv=-57;cid=testClusterID;nsid=759010943;c=1732692997009), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T07:36:40,302 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd 2024-11-27T07:36:40,400 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/cluster_2a34800c-648a-d9f7-feb3-2a3237ffb0ee/zookeeper_0, clientPort=56176, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/cluster_2a34800c-648a-d9f7-feb3-2a3237ffb0ee/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/cluster_2a34800c-648a-d9f7-feb3-2a3237ffb0ee/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T07:36:40,412 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56176 2024-11-27T07:36:40,427 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T07:36:40,434 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T07:36:40,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741825_1001 (size=7) 2024-11-27T07:36:41,090 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c with version=8 2024-11-27T07:36:41,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/hbase-staging 2024-11-27T07:36:41,168 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T07:36:41,362 INFO [Time-limited test {}] client.ConnectionUtils(128): master/289954c9202d:0 server-side Connection retries=9 2024-11-27T07:36:41,371 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T07:36:41,371 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T07:36:41,375 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T07:36:41,375 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T07:36:41,376 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T07:36:41,499 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T07:36:41,553 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T07:36:41,563 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T07:36:41,566 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T07:36:41,593 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 114437 (auto-detected) 2024-11-27T07:36:41,594 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-27T07:36:41,613 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41501 2024-11-27T07:36:41,636 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41501 connecting to ZooKeeper ensemble=127.0.0.1:56176 2024-11-27T07:36:41,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:415010x0, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T07:36:41,761 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41501-0x1017b38fc630000 connected 2024-11-27T07:36:42,237 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T07:36:42,241 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T07:36:42,253 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T07:36:42,259 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c, hbase.cluster.distributed=false 2024-11-27T07:36:42,286 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T07:36:42,290 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41501 2024-11-27T07:36:42,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41501 2024-11-27T07:36:42,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41501 2024-11-27T07:36:42,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41501 2024-11-27T07:36:42,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41501 2024-11-27T07:36:42,403 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/289954c9202d:0 server-side Connection retries=9 2024-11-27T07:36:42,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T07:36:42,406 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T07:36:42,406 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T07:36:42,406 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T07:36:42,406 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T07:36:42,410 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T07:36:42,413 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T07:36:42,415 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41527 2024-11-27T07:36:42,417 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41527 connecting to ZooKeeper ensemble=127.0.0.1:56176 2024-11-27T07:36:42,419 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T07:36:42,422 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T07:36:42,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415270x0, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T07:36:42,444 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T07:36:42,444 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41527-0x1017b38fc630001 connected 2024-11-27T07:36:42,449 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T07:36:42,458 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T07:36:42,460 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T07:36:42,465 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T07:36:42,466 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41527 2024-11-27T07:36:42,466 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41527 2024-11-27T07:36:42,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41527 2024-11-27T07:36:42,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41527 2024-11-27T07:36:42,468 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41527 2024-11-27T07:36:42,482 DEBUG [M:0;289954c9202d:41501 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;289954c9202d:41501 2024-11-27T07:36:42,483 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/289954c9202d,41501,1732693001213 2024-11-27T07:36:42,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T07:36:42,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T07:36:42,615 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/289954c9202d,41501,1732693001213 2024-11-27T07:36:42,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T07:36:42,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:36:42,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:36:42,683 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T07:36:42,685 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/289954c9202d,41501,1732693001213 from backup master directory 2024-11-27T07:36:42,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T07:36:42,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/289954c9202d,41501,1732693001213 2024-11-27T07:36:42,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T07:36:42,769 WARN [master/289954c9202d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T07:36:42,770 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=289954c9202d,41501,1732693001213 2024-11-27T07:36:42,774 INFO [master/289954c9202d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T07:36:42,776 INFO [master/289954c9202d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T07:36:42,829 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/hbase.id] with ID: 6b293ee0-2001-4e0c-a423-4b513843e53d 2024-11-27T07:36:42,829 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/.tmp/hbase.id 2024-11-27T07:36:42,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741826_1002 (size=42) 2024-11-27T07:36:43,240 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/.tmp/hbase.id]:[hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/hbase.id] 2024-11-27T07:36:43,283 INFO [master/289954c9202d:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T07:36:43,287 INFO [master/289954c9202d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T07:36:43,305 INFO [master/289954c9202d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-27T07:36:43,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:36:43,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:36:43,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741827_1003 (size=196) 2024-11-27T07:36:43,758 INFO [master/289954c9202d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:36:43,761 INFO [master/289954c9202d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T07:36:43,781 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:36:43,785 INFO [master/289954c9202d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T07:36:43,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741828_1004 (size=1189) 2024-11-27T07:36:44,240 INFO [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store 2024-11-27T07:36:44,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741829_1005 (size=34) 2024-11-27T07:36:44,669 INFO [master/289954c9202d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T07:36:44,676 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:36:44,677 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T07:36:44,677 INFO [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:36:44,678 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:36:44,679 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T07:36:44,679 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:36:44,680 INFO [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:36:44,681 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732693004677Disabling compacts and flushes for region at 1732693004677Disabling writes for close at 1732693004679 (+2 ms)Writing region close event to WAL at 1732693004679Closed at 1732693004680 (+1 ms) 2024-11-27T07:36:44,684 WARN [master/289954c9202d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/.initializing 2024-11-27T07:36:44,684 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/WALs/289954c9202d,41501,1732693001213 2024-11-27T07:36:44,694 INFO [master/289954c9202d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T07:36:44,708 INFO [master/289954c9202d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=289954c9202d%2C41501%2C1732693001213, suffix=, logDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/WALs/289954c9202d,41501,1732693001213, archiveDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/oldWALs, maxLogs=10 2024-11-27T07:36:44,731 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/WALs/289954c9202d,41501,1732693001213/289954c9202d%2C41501%2C1732693001213.1732693004713, exclude list is [], retry=0 2024-11-27T07:36:44,747 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46455,DS-82256989-cc76-4a34-88fc-db2c26fc19a1,DISK] 2024-11-27T07:36:44,750 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T07:36:44,781 INFO [master/289954c9202d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/WALs/289954c9202d,41501,1732693001213/289954c9202d%2C41501%2C1732693001213.1732693004713 2024-11-27T07:36:44,782 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40819:40819)] 2024-11-27T07:36:44,783 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:36:44,783 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:36:44,786 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:36:44,787 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:36:44,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:36:44,842 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T07:36:44,845 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:36:44,847 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:36:44,848 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:36:44,851 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T07:36:44,851 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:36:44,852 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:36:44,852 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:36:44,855 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T07:36:44,855 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:36:44,856 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:36:44,856 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:36:44,859 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T07:36:44,859 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:36:44,860 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:36:44,860 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:36:44,864 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:36:44,865 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:36:44,870 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:36:44,871 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:36:44,875 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T07:36:44,879 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:36:44,883 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:36:44,885 INFO [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61710053, jitterRate=-0.08044855296611786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T07:36:44,892 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732693004798Initializing all the Stores at 1732693004800 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732693004800Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693004801 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693004801Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693004801Cleaning up temporary data from old regions at 1732693004871 (+70 ms)Region opened successfully at 1732693004892 (+21 ms) 2024-11-27T07:36:44,893 INFO [master/289954c9202d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T07:36:44,924 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a60a0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=289954c9202d/172.17.0.3:0 2024-11-27T07:36:44,960 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T07:36:44,970 INFO [master/289954c9202d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T07:36:44,970 INFO [master/289954c9202d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T07:36:44,973 INFO [master/289954c9202d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T07:36:44,974 INFO [master/289954c9202d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T07:36:44,979 INFO [master/289954c9202d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-27T07:36:44,979 INFO [master/289954c9202d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T07:36:45,002 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T07:36:45,010 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T07:36:45,028 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T07:36:45,031 INFO [master/289954c9202d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T07:36:45,033 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T07:36:45,043 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T07:36:45,045 INFO [master/289954c9202d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T07:36:45,049 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T07:36:45,060 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T07:36:45,062 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T07:36:45,074 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T07:36:45,098 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T07:36:45,106 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T07:36:45,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T07:36:45,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T07:36:45,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:36:45,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:36:45,119 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=289954c9202d,41501,1732693001213, sessionid=0x1017b38fc630000, setting cluster-up flag (Was=false) 2024-11-27T07:36:45,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:36:45,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:36:45,180 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T07:36:45,182 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=289954c9202d,41501,1732693001213 2024-11-27T07:36:45,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:36:45,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:36:45,260 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T07:36:45,263 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=289954c9202d,41501,1732693001213 2024-11-27T07:36:45,272 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T07:36:45,274 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(746): ClusterId : 6b293ee0-2001-4e0c-a423-4b513843e53d 2024-11-27T07:36:45,276 DEBUG [RS:0;289954c9202d:41527 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T07:36:45,334 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T07:36:45,342 INFO [master/289954c9202d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T07:36:45,349 INFO [master/289954c9202d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T07:36:45,349 DEBUG [RS:0;289954c9202d:41527 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T07:36:45,350 DEBUG [RS:0;289954c9202d:41527 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T07:36:45,354 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 289954c9202d,41501,1732693001213 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T07:36:45,392 DEBUG [RS:0;289954c9202d:41527 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T07:36:45,392 DEBUG [RS:0;289954c9202d:41527 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ec825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=289954c9202d/172.17.0.3:0 2024-11-27T07:36:45,393 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/289954c9202d:0, corePoolSize=5, maxPoolSize=5 2024-11-27T07:36:45,393 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/289954c9202d:0, corePoolSize=5, maxPoolSize=5 2024-11-27T07:36:45,393 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/289954c9202d:0, corePoolSize=5, maxPoolSize=5 2024-11-27T07:36:45,393 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/289954c9202d:0, corePoolSize=5, maxPoolSize=5 2024-11-27T07:36:45,394 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/289954c9202d:0, corePoolSize=10, maxPoolSize=10 2024-11-27T07:36:45,394 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:36:45,394 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/289954c9202d:0, corePoolSize=2, maxPoolSize=2 2024-11-27T07:36:45,394 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:36:45,399 INFO [master/289954c9202d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732693035399 2024-11-27T07:36:45,401 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T07:36:45,401 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T07:36:45,401 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T07:36:45,402 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T07:36:45,406 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T07:36:45,407 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T07:36:45,407 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T07:36:45,407 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T07:36:45,407 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:36:45,408 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T07:36:45,409 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;289954c9202d:41527 2024-11-27T07:36:45,410 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T07:36:45,414 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T07:36:45,414 INFO [RS:0;289954c9202d:41527 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T07:36:45,414 INFO [RS:0;289954c9202d:41527 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T07:36:45,415 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T07:36:45,415 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T07:36:45,416 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T07:36:45,418 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(2659): reportForDuty to master=289954c9202d,41501,1732693001213 with port=41527, startcode=1732693002367 2024-11-27T07:36:45,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741831_1007 (size=1321) 2024-11-27T07:36:45,420 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T07:36:45,421 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T07:36:45,425 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/289954c9202d:0:becomeActiveMaster-HFileCleaner.large.0-1732693005422,5,FailOnTimeoutGroup] 2024-11-27T07:36:45,426 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/289954c9202d:0:becomeActiveMaster-HFileCleaner.small.0-1732693005426,5,FailOnTimeoutGroup] 2024-11-27T07:36:45,426 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T07:36:45,426 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T07:36:45,427 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T07:36:45,428 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T07:36:45,431 DEBUG [RS:0;289954c9202d:41527 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T07:36:45,464 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52613, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T07:36:45,469 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41501 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 289954c9202d,41527,1732693002367 2024-11-27T07:36:45,471 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41501 {}] master.ServerManager(517): Registering regionserver=289954c9202d,41527,1732693002367 2024-11-27T07:36:45,485 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c 2024-11-27T07:36:45,486 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46073 2024-11-27T07:36:45,486 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T07:36:45,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T07:36:45,503 DEBUG [RS:0;289954c9202d:41527 {}] zookeeper.ZKUtil(111): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/289954c9202d,41527,1732693002367 2024-11-27T07:36:45,504 WARN [RS:0;289954c9202d:41527 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T07:36:45,504 INFO [RS:0;289954c9202d:41527 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T07:36:45,504 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/WALs/289954c9202d,41527,1732693002367 2024-11-27T07:36:45,507 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [289954c9202d,41527,1732693002367] 2024-11-27T07:36:45,531 INFO [RS:0;289954c9202d:41527 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T07:36:45,547 INFO [RS:0;289954c9202d:41527 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T07:36:45,553 INFO [RS:0;289954c9202d:41527 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T07:36:45,553 INFO [RS:0;289954c9202d:41527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T07:36:45,554 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T07:36:45,559 INFO [RS:0;289954c9202d:41527 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T07:36:45,560 INFO [RS:0;289954c9202d:41527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T07:36:45,561 DEBUG [RS:0;289954c9202d:41527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:36:45,561 DEBUG [RS:0;289954c9202d:41527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:36:45,561 DEBUG [RS:0;289954c9202d:41527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:36:45,561 DEBUG [RS:0;289954c9202d:41527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:36:45,561 DEBUG [RS:0;289954c9202d:41527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:36:45,562 DEBUG [RS:0;289954c9202d:41527 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/289954c9202d:0, corePoolSize=2, maxPoolSize=2 2024-11-27T07:36:45,562 DEBUG [RS:0;289954c9202d:41527 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:36:45,562 DEBUG [RS:0;289954c9202d:41527 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:36:45,562 DEBUG [RS:0;289954c9202d:41527 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:36:45,562 DEBUG [RS:0;289954c9202d:41527 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:36:45,562 DEBUG [RS:0;289954c9202d:41527 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:36:45,562 DEBUG [RS:0;289954c9202d:41527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:36:45,563 DEBUG [RS:0;289954c9202d:41527 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/289954c9202d:0, corePoolSize=3, maxPoolSize=3 2024-11-27T07:36:45,563 DEBUG [RS:0;289954c9202d:41527 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0, corePoolSize=3, maxPoolSize=3 2024-11-27T07:36:45,564 INFO [RS:0;289954c9202d:41527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T07:36:45,564 INFO [RS:0;289954c9202d:41527 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T07:36:45,564 INFO [RS:0;289954c9202d:41527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T07:36:45,564 INFO [RS:0;289954c9202d:41527 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T07:36:45,564 INFO [RS:0;289954c9202d:41527 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T07:36:45,564 INFO [RS:0;289954c9202d:41527 {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,41527,1732693002367-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T07:36:45,585 INFO [RS:0;289954c9202d:41527 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T07:36:45,587 INFO [RS:0;289954c9202d:41527 {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,41527,1732693002367-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T07:36:45,588 INFO [RS:0;289954c9202d:41527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T07:36:45,588 INFO [RS:0;289954c9202d:41527 {}] regionserver.Replication(171): 289954c9202d,41527,1732693002367 started 2024-11-27T07:36:45,610 INFO [RS:0;289954c9202d:41527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T07:36:45,610 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1482): Serving as 289954c9202d,41527,1732693002367, RpcServer on 289954c9202d/172.17.0.3:41527, sessionid=0x1017b38fc630001 2024-11-27T07:36:45,611 DEBUG [RS:0;289954c9202d:41527 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T07:36:45,611 DEBUG [RS:0;289954c9202d:41527 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 289954c9202d,41527,1732693002367 2024-11-27T07:36:45,611 DEBUG [RS:0;289954c9202d:41527 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '289954c9202d,41527,1732693002367' 2024-11-27T07:36:45,612 DEBUG [RS:0;289954c9202d:41527 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T07:36:45,613 DEBUG [RS:0;289954c9202d:41527 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T07:36:45,614 DEBUG [RS:0;289954c9202d:41527 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T07:36:45,614 DEBUG [RS:0;289954c9202d:41527 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T07:36:45,614 DEBUG [RS:0;289954c9202d:41527 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 289954c9202d,41527,1732693002367 2024-11-27T07:36:45,614 DEBUG [RS:0;289954c9202d:41527 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '289954c9202d,41527,1732693002367' 2024-11-27T07:36:45,614 DEBUG [RS:0;289954c9202d:41527 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T07:36:45,615 DEBUG [RS:0;289954c9202d:41527 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T07:36:45,616 DEBUG [RS:0;289954c9202d:41527 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T07:36:45,616 INFO [RS:0;289954c9202d:41527 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T07:36:45,616 INFO [RS:0;289954c9202d:41527 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T07:36:45,822 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T07:36:45,822 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c 2024-11-27T07:36:45,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741832_1008 (size=32) 2024-11-27T07:36:46,234 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:36:46,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T07:36:46,242 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T07:36:46,242 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:36:46,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:36:46,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T07:36:46,247 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T07:36:46,248 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:36:46,249 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:36:46,249 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T07:36:46,253 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T07:36:46,253 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:36:46,254 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:36:46,255 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T07:36:46,259 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T07:36:46,259 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:36:46,260 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:36:46,261 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T07:36:46,263 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740 2024-11-27T07:36:46,264 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740 2024-11-27T07:36:46,269 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T07:36:46,269 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T07:36:46,270 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T07:36:46,274 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T07:36:46,280 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:36:46,282 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62831163, jitterRate=-0.06374271214008331}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T07:36:46,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732693006235Initializing all the Stores at 1732693006237 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732693006237Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732693006238 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693006238Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732693006238Cleaning up temporary data from old regions at 1732693006269 (+31 ms)Region opened successfully at 1732693006285 (+16 ms) 2024-11-27T07:36:46,286 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T07:36:46,286 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T07:36:46,286 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T07:36:46,286 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T07:36:46,286 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T07:36:46,288 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T07:36:46,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732693006285Disabling compacts and flushes for region at 1732693006285Disabling writes for close at 1732693006286 (+1 ms)Writing region close event to WAL at 1732693006287 (+1 ms)Closed at 1732693006288 (+1 ms) 2024-11-27T07:36:46,295 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T07:36:46,295 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T07:36:46,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T07:36:46,314 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T07:36:46,321 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T07:36:46,473 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:36:47,473 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:36:48,474 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:36:49,475 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:36:50,476 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:36:51,476 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:36:51,550 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T07:36:51,551 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T07:36:52,477 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:36:53,478 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:36:54,479 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:36:55,479 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:36:56,480 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:36:57,481 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:36:58,482 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:36:59,483 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:00,484 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:01,484 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:02,485 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:03,486 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:04,487 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:05,488 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:06,489 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:07,490 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:08,491 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:09,491 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:10,298 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T07:37:10,492 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:11,493 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:12,494 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:13,495 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:14,495 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:15,496 WARN [289954c9202d:41501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T07:37:15,623 INFO [RS:0;289954c9202d:41527 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T07:37:15,628 INFO [RS:0;289954c9202d:41527 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=289954c9202d%2C41527%2C1732693002367, suffix=, logDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/WALs/289954c9202d,41527,1732693002367, archiveDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/oldWALs, maxLogs=32 2024-11-27T07:37:15,645 DEBUG [RS:0;289954c9202d:41527 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/WALs/289954c9202d,41527,1732693002367/289954c9202d%2C41527%2C1732693002367.1732693035632, exclude list is [], retry=0 2024-11-27T07:37:15,649 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46455,DS-82256989-cc76-4a34-88fc-db2c26fc19a1,DISK] 2024-11-27T07:37:15,653 INFO [RS:0;289954c9202d:41527 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/WALs/289954c9202d,41527,1732693002367/289954c9202d%2C41527%2C1732693002367.1732693035632 2024-11-27T07:37:15,654 DEBUG [RS:0;289954c9202d:41527 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40819:40819)] 2024-11-27T07:37:15,753 DEBUG [289954c9202d:41501 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-27T07:37:15,764 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:15,771 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 289954c9202d,41527,1732693002367, state=OPENING 2024-11-27T07:37:15,819 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T07:37:15,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:37:15,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:37:15,831 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T07:37:15,831 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T07:37:15,833 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T07:37:15,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=289954c9202d,41527,1732693002367}] 2024-11-27T07:37:16,017 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T07:37:16,020 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46939, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T07:37:16,031 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T07:37:16,031 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T07:37:16,032 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T07:37:16,035 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=289954c9202d%2C41527%2C1732693002367.meta, suffix=.meta, logDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/WALs/289954c9202d,41527,1732693002367, archiveDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/oldWALs, maxLogs=32 2024-11-27T07:37:16,051 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/WALs/289954c9202d,41527,1732693002367/289954c9202d%2C41527%2C1732693002367.meta.1732693036037.meta, exclude list is [], retry=0 2024-11-27T07:37:16,054 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46455,DS-82256989-cc76-4a34-88fc-db2c26fc19a1,DISK] 2024-11-27T07:37:16,057 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/WALs/289954c9202d,41527,1732693002367/289954c9202d%2C41527%2C1732693002367.meta.1732693036037.meta 2024-11-27T07:37:16,057 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40819:40819)] 2024-11-27T07:37:16,058 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:37:16,059 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T07:37:16,061 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T07:37:16,065 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T07:37:16,069 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T07:37:16,070 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:16,070 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T07:37:16,070 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T07:37:16,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T07:37:16,074 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T07:37:16,074 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:16,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:37:16,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T07:37:16,077 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T07:37:16,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:16,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:37:16,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T07:37:16,079 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T07:37:16,079 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:16,080 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:37:16,080 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T07:37:16,082 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T07:37:16,082 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:16,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:37:16,083 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T07:37:16,084 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740 2024-11-27T07:37:16,087 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740 2024-11-27T07:37:16,089 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T07:37:16,089 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T07:37:16,090 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T07:37:16,092 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T07:37:16,094 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61720342, jitterRate=-0.08029523491859436}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T07:37:16,094 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T07:37:16,095 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732693036070Writing region info on filesystem at 1732693036070Initializing all the Stores at 1732693036072 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732693036072Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732693036072Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693036072Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732693036072Cleaning up temporary data from old regions at 1732693036089 (+17 ms)Running coprocessor post-open hooks at 1732693036094 (+5 ms)Region opened successfully at 1732693036095 (+1 ms) 2024-11-27T07:37:16,101 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732693036006 2024-11-27T07:37:16,113 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T07:37:16,113 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T07:37:16,115 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:16,117 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 289954c9202d,41527,1732693002367, state=OPEN 2024-11-27T07:37:16,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T07:37:16,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T07:37:16,156 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T07:37:16,156 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T07:37:16,156 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=289954c9202d,41527,1732693002367 2024-11-27T07:37:16,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T07:37:16,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=289954c9202d,41527,1732693002367 in 322 msec 2024-11-27T07:37:16,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T07:37:16,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.8620 sec 2024-11-27T07:37:16,172 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T07:37:16,172 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T07:37:16,189 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:37:16,190 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,41527,1732693002367, seqNum=-1] 2024-11-27T07:37:16,209 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:37:16,212 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35747, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:37:16,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.9320 sec 2024-11-27T07:37:16,231 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732693036231, completionTime=-1 2024-11-27T07:37:16,234 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-27T07:37:16,234 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T07:37:16,259 INFO [master/289954c9202d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-27T07:37:16,259 INFO [master/289954c9202d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732693096259 2024-11-27T07:37:16,259 INFO [master/289954c9202d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732693156259 2024-11-27T07:37:16,259 INFO [master/289954c9202d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-27T07:37:16,261 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,41501,1732693001213-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T07:37:16,262 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,41501,1732693001213-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T07:37:16,262 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,41501,1732693001213-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T07:37:16,282 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-289954c9202d:41501, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T07:37:16,282 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T07:37:16,283 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T07:37:16,289 DEBUG [master/289954c9202d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T07:37:16,316 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.545sec 2024-11-27T07:37:16,317 INFO [master/289954c9202d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T07:37:16,318 INFO [master/289954c9202d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T07:37:16,320 INFO [master/289954c9202d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T07:37:16,320 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T07:37:16,321 INFO [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T07:37:16,321 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,41501,1732693001213-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T07:37:16,322 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,41501,1732693001213-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T07:37:16,331 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T07:37:16,332 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T07:37:16,332 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,41501,1732693001213-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T07:37:16,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19040525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:37:16,421 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T07:37:16,421 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T07:37:16,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,41501,-1 for getting cluster id 2024-11-27T07:37:16,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:37:16,432 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b293ee0-2001-4e0c-a423-4b513843e53d' 2024-11-27T07:37:16,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:37:16,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b293ee0-2001-4e0c-a423-4b513843e53d" 2024-11-27T07:37:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6efcb2cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:37:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,41501,-1] 2024-11-27T07:37:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:37:16,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:37:16,443 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:37:16,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ae8a796, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:37:16,446 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:37:16,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,41527,1732693002367, seqNum=-1] 2024-11-27T07:37:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:37:16,455 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:37:16,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=289954c9202d,41501,1732693001213 2024-11-27T07:37:16,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b4529cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:37:16,474 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,41501,-1 for getting cluster id 2024-11-27T07:37:16,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:37:16,476 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6b293ee0-2001-4e0c-a423-4b513843e53d' 2024-11-27T07:37:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:37:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6b293ee0-2001-4e0c-a423-4b513843e53d" 2024-11-27T07:37:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5082d9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:37:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,41501,-1] 2024-11-27T07:37:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:37:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:37:16,479 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:37:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad2d8e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:37:16,491 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=278, ProcessCount=11, AvailableMemoryMB=5589 2024-11-27T07:37:16,496 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T07:37:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 289954c9202d,41501,1732693001213 2024-11-27T07:37:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1a1132fb 2024-11-27T07:37:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T07:37:16,505 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T07:37:16,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:37:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-27T07:37:16,521 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:37:16,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-27T07:37:16,524 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:16,528 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:37:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T07:37:16,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741835_1011 (size=428) 2024-11-27T07:37:16,948 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2c42380c364991ffc7b56aaffd602a09, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c 2024-11-27T07:37:16,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741836_1012 (size=79) 2024-11-27T07:37:17,363 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:17,363 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 2c42380c364991ffc7b56aaffd602a09, disabling compactions & flushes 2024-11-27T07:37:17,363 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. 2024-11-27T07:37:17,363 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. 2024-11-27T07:37:17,363 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. after waiting 0 ms 2024-11-27T07:37:17,364 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. 2024-11-27T07:37:17,364 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. 2024-11-27T07:37:17,364 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2c42380c364991ffc7b56aaffd602a09: Waiting for close lock at 1732693037363Disabling compacts and flushes for region at 1732693037363Disabling writes for close at 1732693037363Writing region close event to WAL at 1732693037364 (+1 ms)Closed at 1732693037364 2024-11-27T07:37:17,367 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:37:17,375 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732693037368"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732693037368"}]},"ts":"1732693037368"} 2024-11-27T07:37:17,381 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:37:17,383 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:37:17,385 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693037383"}]},"ts":"1732693037383"} 2024-11-27T07:37:17,389 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-27T07:37:17,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=2c42380c364991ffc7b56aaffd602a09, ASSIGN}] 2024-11-27T07:37:17,394 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=2c42380c364991ffc7b56aaffd602a09, ASSIGN 2024-11-27T07:37:17,396 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=2c42380c364991ffc7b56aaffd602a09, ASSIGN; state=OFFLINE, location=289954c9202d,41527,1732693002367; forceNewPlan=false, retain=false 2024-11-27T07:37:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T07:37:17,548 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2c42380c364991ffc7b56aaffd602a09, regionState=OPENING, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:17,555 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=2c42380c364991ffc7b56aaffd602a09, ASSIGN because future has completed 2024-11-27T07:37:17,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c42380c364991ffc7b56aaffd602a09, server=289954c9202d,41527,1732693002367}] 2024-11-27T07:37:17,718 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. 2024-11-27T07:37:17,719 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 2c42380c364991ffc7b56aaffd602a09, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:37:17,719 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:37:17,719 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:17,719 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:37:17,719 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:37:17,722 INFO [StoreOpener-2c42380c364991ffc7b56aaffd602a09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:37:17,724 INFO [StoreOpener-2c42380c364991ffc7b56aaffd602a09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c42380c364991ffc7b56aaffd602a09 columnFamilyName testFamily 2024-11-27T07:37:17,724 DEBUG [StoreOpener-2c42380c364991ffc7b56aaffd602a09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:17,725 INFO [StoreOpener-2c42380c364991ffc7b56aaffd602a09-1 {}] regionserver.HStore(327): Store=2c42380c364991ffc7b56aaffd602a09/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:37:17,725 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:37:17,727 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:37:17,727 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:37:17,728 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:37:17,728 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:37:17,732 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:37:17,736 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/2c42380c364991ffc7b56aaffd602a09/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:37:17,737 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 2c42380c364991ffc7b56aaffd602a09; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66641188, jitterRate=-0.0069689154624938965}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T07:37:17,737 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:37:17,739 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 2c42380c364991ffc7b56aaffd602a09: Running coprocessor pre-open hook at 1732693037719Writing region info on filesystem at 1732693037719Initializing all the Stores at 1732693037721 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693037721Cleaning up temporary data from old regions at 1732693037728 (+7 ms)Running coprocessor post-open hooks at 1732693037737 (+9 ms)Region opened successfully at 1732693037738 (+1 ms) 2024-11-27T07:37:17,741 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09., pid=6, masterSystemTime=1732693037712 2024-11-27T07:37:17,745 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. 2024-11-27T07:37:17,745 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. 2024-11-27T07:37:17,746 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2c42380c364991ffc7b56aaffd602a09, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:17,750 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c42380c364991ffc7b56aaffd602a09, server=289954c9202d,41527,1732693002367 because future has completed 2024-11-27T07:37:17,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T07:37:17,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2c42380c364991ffc7b56aaffd602a09, server=289954c9202d,41527,1732693002367 in 196 msec 2024-11-27T07:37:17,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T07:37:17,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=2c42380c364991ffc7b56aaffd602a09, ASSIGN in 365 msec 2024-11-27T07:37:17,762 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:37:17,762 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693037762"}]},"ts":"1732693037762"} 2024-11-27T07:37:17,765 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-27T07:37:17,767 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:37:17,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2540 sec 2024-11-27T07:37:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T07:37:19,551 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-27T07:37:19,551 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-27T07:37:19,552 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:37:19,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-27T07:37:19,560 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:37:19,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-27T07:37:19,564 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:37:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,41527,1732693002367, seqNum=-1] 2024-11-27T07:37:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:37:19,569 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:37:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09., hostname=289954c9202d,41527,1732693002367, seqNum=2] 2024-11-27T07:37:19,589 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T07:37:19,593 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-27T07:37:19,597 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-27T07:37:19,601 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-27T07:37:19,603 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-27T07:37:19,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-4569127999240142840 2024-11-27T07:37:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:34646 deadline: 1732693042603, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T07:37:19,629 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-27T07:37:19,630 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-27T07:37:19,631 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T07:37:19,633 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T07:37:19,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-4569127999240142838 2024-11-27T07:37:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:34646 deadline: 1732693042632, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T07:37:19,638 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T07:37:19,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-4569127999240142837 2024-11-27T07:37:19,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:34646 deadline: 1732693042637, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T07:37:19,643 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T07:37:19,644 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-4569127999240142836 2024-11-27T07:37:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:34646 deadline: 1732693042642, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T07:37:19,648 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T07:37:19,657 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/289954c9202d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1658004170_22 at /127.0.0.1:48450 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=280 (was 278) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5528 (was 5589) 2024-11-27T07:37:19,665 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=280, ProcessCount=11, AvailableMemoryMB=5527 2024-11-27T07:37:19,669 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:37:19,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-27T07:37:19,672 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:37:19,672 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:19,672 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-27T07:37:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T07:37:19,674 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:37:19,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741837_1013 (size=414) 2024-11-27T07:37:20,091 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0720db5474d7242cd64e0ebf2bd06ad8, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c 2024-11-27T07:37:20,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741838_1014 (size=65) 2024-11-27T07:37:20,504 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:20,504 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 0720db5474d7242cd64e0ebf2bd06ad8, disabling compactions & flushes 2024-11-27T07:37:20,504 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. 2024-11-27T07:37:20,504 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. 2024-11-27T07:37:20,505 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. after waiting 0 ms 2024-11-27T07:37:20,505 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. 2024-11-27T07:37:20,505 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. 2024-11-27T07:37:20,505 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0720db5474d7242cd64e0ebf2bd06ad8: Waiting for close lock at 1732693040504Disabling compacts and flushes for region at 1732693040504Disabling writes for close at 1732693040505 (+1 ms)Writing region close event to WAL at 1732693040505Closed at 1732693040505 2024-11-27T07:37:20,511 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:37:20,511 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732693040511"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732693040511"}]},"ts":"1732693040511"} 2024-11-27T07:37:20,515 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:37:20,517 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:37:20,518 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693040517"}]},"ts":"1732693040517"} 2024-11-27T07:37:20,521 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-27T07:37:20,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0720db5474d7242cd64e0ebf2bd06ad8, ASSIGN}] 2024-11-27T07:37:20,524 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0720db5474d7242cd64e0ebf2bd06ad8, ASSIGN 2024-11-27T07:37:20,525 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0720db5474d7242cd64e0ebf2bd06ad8, ASSIGN; state=OFFLINE, location=289954c9202d,41527,1732693002367; forceNewPlan=false, retain=false 2024-11-27T07:37:20,676 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0720db5474d7242cd64e0ebf2bd06ad8, regionState=OPENING, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:20,681 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0720db5474d7242cd64e0ebf2bd06ad8, ASSIGN because future has completed 2024-11-27T07:37:20,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0720db5474d7242cd64e0ebf2bd06ad8, server=289954c9202d,41527,1732693002367}] 2024-11-27T07:37:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T07:37:20,846 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. 2024-11-27T07:37:20,847 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 0720db5474d7242cd64e0ebf2bd06ad8, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:37:20,848 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 0720db5474d7242cd64e0ebf2bd06ad8 2024-11-27T07:37:20,848 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:20,849 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 0720db5474d7242cd64e0ebf2bd06ad8 2024-11-27T07:37:20,849 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 0720db5474d7242cd64e0ebf2bd06ad8 2024-11-27T07:37:20,852 INFO [StoreOpener-0720db5474d7242cd64e0ebf2bd06ad8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0720db5474d7242cd64e0ebf2bd06ad8 2024-11-27T07:37:20,854 INFO [StoreOpener-0720db5474d7242cd64e0ebf2bd06ad8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0720db5474d7242cd64e0ebf2bd06ad8 columnFamilyName testFamily 2024-11-27T07:37:20,854 DEBUG [StoreOpener-0720db5474d7242cd64e0ebf2bd06ad8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:20,855 INFO [StoreOpener-0720db5474d7242cd64e0ebf2bd06ad8-1 {}] regionserver.HStore(327): Store=0720db5474d7242cd64e0ebf2bd06ad8/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:37:20,856 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 0720db5474d7242cd64e0ebf2bd06ad8 2024-11-27T07:37:20,857 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScanner/0720db5474d7242cd64e0ebf2bd06ad8 2024-11-27T07:37:20,858 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScanner/0720db5474d7242cd64e0ebf2bd06ad8 2024-11-27T07:37:20,859 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 0720db5474d7242cd64e0ebf2bd06ad8 2024-11-27T07:37:20,859 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 0720db5474d7242cd64e0ebf2bd06ad8 2024-11-27T07:37:20,862 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 0720db5474d7242cd64e0ebf2bd06ad8 2024-11-27T07:37:20,865 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScanner/0720db5474d7242cd64e0ebf2bd06ad8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:37:20,866 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 0720db5474d7242cd64e0ebf2bd06ad8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74280477, jitterRate=0.10686536133289337}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T07:37:20,867 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0720db5474d7242cd64e0ebf2bd06ad8 2024-11-27T07:37:20,868 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 0720db5474d7242cd64e0ebf2bd06ad8: Running coprocessor pre-open hook at 1732693040849Writing region info on filesystem at 1732693040849Initializing all the Stores at 1732693040852 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693040852Cleaning up temporary data from old regions at 1732693040859 (+7 ms)Running coprocessor post-open hooks at 1732693040867 (+8 ms)Region opened successfully at 1732693040868 (+1 ms) 2024-11-27T07:37:20,870 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8., pid=9, masterSystemTime=1732693040836 2024-11-27T07:37:20,874 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. 2024-11-27T07:37:20,874 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. 2024-11-27T07:37:20,875 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0720db5474d7242cd64e0ebf2bd06ad8, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:20,879 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0720db5474d7242cd64e0ebf2bd06ad8, server=289954c9202d,41527,1732693002367 because future has completed 2024-11-27T07:37:20,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-27T07:37:20,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 0720db5474d7242cd64e0ebf2bd06ad8, server=289954c9202d,41527,1732693002367 in 200 msec 2024-11-27T07:37:20,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T07:37:20,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0720db5474d7242cd64e0ebf2bd06ad8, ASSIGN in 364 msec 2024-11-27T07:37:20,891 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:37:20,891 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693040891"}]},"ts":"1732693040891"} 2024-11-27T07:37:20,895 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-27T07:37:20,896 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:37:20,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2280 sec 2024-11-27T07:37:21,609 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:37:21,636 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-27T07:37:21,637 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T07:37:21,638 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-27T07:37:21,639 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T07:37:21,639 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T07:37:21,640 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T07:37:21,640 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T07:37:21,640 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-27T07:37:21,640 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T07:37:21,641 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-27T07:37:21,641 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-27T07:37:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T07:37:22,701 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-27T07:37:22,701 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-27T07:37:22,702 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:37:22,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-27T07:37:22,708 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:37:22,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-27T07:37:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8., hostname=289954c9202d,41527,1732693002367, seqNum=2] 2024-11-27T07:37:22,727 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T07:37:22,728 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-27T07:37:22,732 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-27T07:37:22,734 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-27T07:37:28,747 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8., hostname=289954c9202d,41527,1732693002367, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8., hostname=289954c9202d,41527,1732693002367, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=289954c9202d:41527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6009ms, rpcTimeout=5999ms 2024-11-27T07:37:28,748 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8., hostname=289954c9202d,41527,1732693002367, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6009ms, rpcTimeout=5999ms 2024-11-27T07:37:28,748 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8., hostname=289954c9202d,41527,1732693002367, seqNum=2 from cache 2024-11-27T07:37:28,757 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 280), ProcessCount=11 (was 11), AvailableMemoryMB=5427 (was 5527) 2024-11-27T07:37:28,765 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=5427 2024-11-27T07:37:28,768 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:37:28,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-27T07:37:28,772 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:37:28,772 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:28,772 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-27T07:37:28,774 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:37:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-27T07:37:28,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741839_1015 (size=412) 2024-11-27T07:37:29,218 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 84e136ef3c363cd7fdbf7742a369e042, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c 2024-11-27T07:37:29,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741840_1016 (size=63) 2024-11-27T07:37:29,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-27T07:37:29,628 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:29,629 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 84e136ef3c363cd7fdbf7742a369e042, disabling compactions & flushes 2024-11-27T07:37:29,629 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. 2024-11-27T07:37:29,629 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. 2024-11-27T07:37:29,629 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. after waiting 0 ms 2024-11-27T07:37:29,629 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. 2024-11-27T07:37:29,629 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. 2024-11-27T07:37:29,630 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 84e136ef3c363cd7fdbf7742a369e042: Waiting for close lock at 1732693049629Disabling compacts and flushes for region at 1732693049629Disabling writes for close at 1732693049629Writing region close event to WAL at 1732693049629Closed at 1732693049629 2024-11-27T07:37:29,633 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:37:29,634 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732693049634"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732693049634"}]},"ts":"1732693049634"} 2024-11-27T07:37:29,640 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:37:29,642 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:37:29,642 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693049642"}]},"ts":"1732693049642"} 2024-11-27T07:37:29,645 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-27T07:37:29,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=84e136ef3c363cd7fdbf7742a369e042, ASSIGN}] 2024-11-27T07:37:29,647 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=84e136ef3c363cd7fdbf7742a369e042, ASSIGN 2024-11-27T07:37:29,648 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=84e136ef3c363cd7fdbf7742a369e042, ASSIGN; state=OFFLINE, location=289954c9202d,41527,1732693002367; forceNewPlan=false, retain=false 2024-11-27T07:37:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-27T07:37:29,799 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=84e136ef3c363cd7fdbf7742a369e042, regionState=OPENING, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:29,806 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=84e136ef3c363cd7fdbf7742a369e042, ASSIGN because future has completed 2024-11-27T07:37:29,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 84e136ef3c363cd7fdbf7742a369e042, server=289954c9202d,41527,1732693002367}] 2024-11-27T07:37:29,975 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. 2024-11-27T07:37:29,976 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 84e136ef3c363cd7fdbf7742a369e042, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:37:29,976 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 84e136ef3c363cd7fdbf7742a369e042 2024-11-27T07:37:29,976 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:29,977 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 84e136ef3c363cd7fdbf7742a369e042 2024-11-27T07:37:29,977 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 84e136ef3c363cd7fdbf7742a369e042 2024-11-27T07:37:29,980 INFO [StoreOpener-84e136ef3c363cd7fdbf7742a369e042-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 84e136ef3c363cd7fdbf7742a369e042 2024-11-27T07:37:29,983 INFO [StoreOpener-84e136ef3c363cd7fdbf7742a369e042-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84e136ef3c363cd7fdbf7742a369e042 columnFamilyName testFamily 2024-11-27T07:37:29,984 DEBUG [StoreOpener-84e136ef3c363cd7fdbf7742a369e042-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:29,984 INFO [StoreOpener-84e136ef3c363cd7fdbf7742a369e042-1 {}] regionserver.HStore(327): Store=84e136ef3c363cd7fdbf7742a369e042/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:37:29,985 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 84e136ef3c363cd7fdbf7742a369e042 2024-11-27T07:37:29,986 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNextAsync/84e136ef3c363cd7fdbf7742a369e042 2024-11-27T07:37:29,987 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNextAsync/84e136ef3c363cd7fdbf7742a369e042 2024-11-27T07:37:29,987 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 84e136ef3c363cd7fdbf7742a369e042 2024-11-27T07:37:29,987 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 84e136ef3c363cd7fdbf7742a369e042 2024-11-27T07:37:29,990 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 84e136ef3c363cd7fdbf7742a369e042 2024-11-27T07:37:29,994 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNextAsync/84e136ef3c363cd7fdbf7742a369e042/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:37:29,995 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 84e136ef3c363cd7fdbf7742a369e042; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62598684, jitterRate=-0.06720691919326782}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T07:37:29,995 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 84e136ef3c363cd7fdbf7742a369e042 2024-11-27T07:37:29,996 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 84e136ef3c363cd7fdbf7742a369e042: Running coprocessor pre-open hook at 1732693049977Writing region info on filesystem at 1732693049977Initializing all the Stores at 1732693049979 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693049979Cleaning up temporary data from old regions at 1732693049987 (+8 ms)Running coprocessor post-open hooks at 1732693049995 (+8 ms)Region opened successfully at 1732693049996 (+1 ms) 2024-11-27T07:37:29,997 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042., pid=12, masterSystemTime=1732693049964 2024-11-27T07:37:30,000 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. 2024-11-27T07:37:30,000 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. 2024-11-27T07:37:30,001 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=84e136ef3c363cd7fdbf7742a369e042, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:30,004 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 84e136ef3c363cd7fdbf7742a369e042, server=289954c9202d,41527,1732693002367 because future has completed 2024-11-27T07:37:30,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-27T07:37:30,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 84e136ef3c363cd7fdbf7742a369e042, server=289954c9202d,41527,1732693002367 in 198 msec 2024-11-27T07:37:30,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-27T07:37:30,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=84e136ef3c363cd7fdbf7742a369e042, ASSIGN in 364 msec 2024-11-27T07:37:30,014 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:37:30,015 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693050014"}]},"ts":"1732693050014"} 2024-11-27T07:37:30,018 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-27T07:37:30,019 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:37:30,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2500 sec 2024-11-27T07:37:31,550 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-27T07:37:31,550 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T07:37:31,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-27T07:37:31,801 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-27T07:37:31,801 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-27T07:37:31,801 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:37:31,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-27T07:37:31,806 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:37:31,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-27T07:37:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042., hostname=289954c9202d,41527,1732693002367, seqNum=2] 2024-11-27T07:37:31,823 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T07:37:31,823 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-27T07:37:31,827 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-27T07:37:31,830 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-27T07:37:31,831 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-27T07:37:31,832 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-27T07:37:33,641 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-27T07:37:35,503 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:37:37,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41527 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4569127999240142827' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-27T07:37:37,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41527 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:34646 deadline: 1732693063845, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4569127999240142827' does not exist 2024-11-27T07:37:37,849 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 289954c9202d,41527,1732693002367 for closing scanner id = -4569127999240142827 for 84e136ef3c363cd7fdbf7742a369e042 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4569127999240142827' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:37:37,851 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=220 (was 260), ProcessCount=11 (was 11), AvailableMemoryMB=5417 (was 5427) 2024-11-27T07:37:37,860 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=220, ProcessCount=11, AvailableMemoryMB=5416 2024-11-27T07:37:37,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:37:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-27T07:37:37,865 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:37:37,865 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:37,866 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-27T07:37:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T07:37:37,867 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:37:37,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741841_1017 (size=423) 2024-11-27T07:37:38,280 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 282935bd9dd931fa45e861f636eef90f, NAME => 'testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c 2024-11-27T07:37:38,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741842_1018 (size=74) 2024-11-27T07:37:38,292 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:38,293 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 282935bd9dd931fa45e861f636eef90f, disabling compactions & flushes 2024-11-27T07:37:38,293 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. 2024-11-27T07:37:38,293 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. 2024-11-27T07:37:38,293 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. after waiting 0 ms 2024-11-27T07:37:38,293 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. 2024-11-27T07:37:38,293 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. 2024-11-27T07:37:38,293 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 282935bd9dd931fa45e861f636eef90f: Waiting for close lock at 1732693058293Disabling compacts and flushes for region at 1732693058293Disabling writes for close at 1732693058293Writing region close event to WAL at 1732693058293Closed at 1732693058293 2024-11-27T07:37:38,295 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:37:38,295 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732693058295"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732693058295"}]},"ts":"1732693058295"} 2024-11-27T07:37:38,298 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:37:38,299 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:37:38,299 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693058299"}]},"ts":"1732693058299"} 2024-11-27T07:37:38,302 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-27T07:37:38,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=282935bd9dd931fa45e861f636eef90f, ASSIGN}] 2024-11-27T07:37:38,304 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=282935bd9dd931fa45e861f636eef90f, ASSIGN 2024-11-27T07:37:38,306 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=282935bd9dd931fa45e861f636eef90f, ASSIGN; state=OFFLINE, location=289954c9202d,41527,1732693002367; forceNewPlan=false, retain=false 2024-11-27T07:37:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-27T07:37:38,457 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=282935bd9dd931fa45e861f636eef90f, regionState=OPENING, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:38,463 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=282935bd9dd931fa45e861f636eef90f, ASSIGN because future has completed 2024-11-27T07:37:38,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 282935bd9dd931fa45e861f636eef90f, server=289954c9202d,41527,1732693002367}] 2024-11-27T07:37:38,633 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. 2024-11-27T07:37:38,634 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 282935bd9dd931fa45e861f636eef90f, NAME => 'testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:37:38,634 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 282935bd9dd931fa45e861f636eef90f 2024-11-27T07:37:38,634 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:38,635 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 282935bd9dd931fa45e861f636eef90f 2024-11-27T07:37:38,635 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 282935bd9dd931fa45e861f636eef90f 2024-11-27T07:37:38,637 INFO [StoreOpener-282935bd9dd931fa45e861f636eef90f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 282935bd9dd931fa45e861f636eef90f 2024-11-27T07:37:38,641 INFO [StoreOpener-282935bd9dd931fa45e861f636eef90f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 282935bd9dd931fa45e861f636eef90f columnFamilyName testFamily 2024-11-27T07:37:38,642 DEBUG [StoreOpener-282935bd9dd931fa45e861f636eef90f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:38,643 INFO [StoreOpener-282935bd9dd931fa45e861f636eef90f-1 {}] regionserver.HStore(327): Store=282935bd9dd931fa45e861f636eef90f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:37:38,643 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 282935bd9dd931fa45e861f636eef90f 2024-11-27T07:37:38,644 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextException/282935bd9dd931fa45e861f636eef90f 2024-11-27T07:37:38,645 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextException/282935bd9dd931fa45e861f636eef90f 2024-11-27T07:37:38,646 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 282935bd9dd931fa45e861f636eef90f 2024-11-27T07:37:38,647 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 282935bd9dd931fa45e861f636eef90f 2024-11-27T07:37:38,650 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 282935bd9dd931fa45e861f636eef90f 2024-11-27T07:37:38,655 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextException/282935bd9dd931fa45e861f636eef90f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:37:38,656 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 282935bd9dd931fa45e861f636eef90f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64459002, jitterRate=-0.03948602080345154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T07:37:38,656 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 282935bd9dd931fa45e861f636eef90f 2024-11-27T07:37:38,656 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 282935bd9dd931fa45e861f636eef90f: Running coprocessor pre-open hook at 1732693058635Writing region info on filesystem at 1732693058635Initializing all the Stores at 1732693058637 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693058637Cleaning up temporary data from old regions at 1732693058647 (+10 ms)Running coprocessor post-open hooks at 1732693058656 (+9 ms)Region opened successfully at 1732693058656 2024-11-27T07:37:38,658 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f., pid=15, masterSystemTime=1732693058620 2024-11-27T07:37:38,661 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. 2024-11-27T07:37:38,661 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. 2024-11-27T07:37:38,662 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=282935bd9dd931fa45e861f636eef90f, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:38,664 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 282935bd9dd931fa45e861f636eef90f, server=289954c9202d,41527,1732693002367 because future has completed 2024-11-27T07:37:38,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-27T07:37:38,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 282935bd9dd931fa45e861f636eef90f, server=289954c9202d,41527,1732693002367 in 202 msec 2024-11-27T07:37:38,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-27T07:37:38,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=282935bd9dd931fa45e861f636eef90f, ASSIGN in 367 msec 2024-11-27T07:37:38,673 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:37:38,674 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693058673"}]},"ts":"1732693058673"} 2024-11-27T07:37:38,676 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-27T07:37:38,677 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:37:38,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 815 msec 2024-11-27T07:37:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T07:37:38,872 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-27T07:37:38,872 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-27T07:37:38,873 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:37:38,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-27T07:37:38,882 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:37:38,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-27T07:37:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f., hostname=289954c9202d,41527,1732693002367, seqNum=2] 2024-11-27T07:37:38,897 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T07:37:38,897 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-27T07:37:38,900 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-27T07:37:38,902 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-27T07:37:38,903 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-27T07:37:38,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-4569127999240142823 2024-11-27T07:37:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:34646 deadline: 1732693061902, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T07:37:38,906 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-27T07:37:38,907 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-27T07:37:38,908 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T07:37:38,910 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T07:37:38,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-4569127999240142821 2024-11-27T07:37:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:34646 deadline: 1732693061909, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T07:37:38,914 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T07:37:38,915 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-4569127999240142820 2024-11-27T07:37:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:34646 deadline: 1732693061914, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T07:37:38,918 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T07:37:38,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-4569127999240142819 2024-11-27T07:37:38,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:34646 deadline: 1732693061917, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T07:37:38,922 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T07:37:38,930 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=226 (was 225) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1658004170_22 at /127.0.0.1:50964 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=220 (was 220), ProcessCount=11 (was 11), AvailableMemoryMB=5409 (was 5416) 2024-11-27T07:37:38,940 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=226, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=220, ProcessCount=11, AvailableMemoryMB=5408 2024-11-27T07:37:38,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:37:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-27T07:37:38,947 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:37:38,947 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:38,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-27T07:37:38,949 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:37:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-27T07:37:38,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741843_1019 (size=418) 2024-11-27T07:37:39,366 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => edb651d448a6733589b5e22423583e86, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c 2024-11-27T07:37:39,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741844_1020 (size=69) 2024-11-27T07:37:39,645 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-27T07:37:39,782 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:39,783 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing edb651d448a6733589b5e22423583e86, disabling compactions & flushes 2024-11-27T07:37:39,783 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. 2024-11-27T07:37:39,783 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. 2024-11-27T07:37:39,783 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. after waiting 0 ms 2024-11-27T07:37:39,783 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. 2024-11-27T07:37:39,784 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. 2024-11-27T07:37:39,784 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for edb651d448a6733589b5e22423583e86: Waiting for close lock at 1732693059783Disabling compacts and flushes for region at 1732693059783Disabling writes for close at 1732693059783Writing region close event to WAL at 1732693059783Closed at 1732693059783 2024-11-27T07:37:39,788 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:37:39,788 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732693059788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732693059788"}]},"ts":"1732693059788"} 2024-11-27T07:37:39,791 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:37:39,793 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:37:39,793 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693059793"}]},"ts":"1732693059793"} 2024-11-27T07:37:39,795 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-27T07:37:39,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=edb651d448a6733589b5e22423583e86, ASSIGN}] 2024-11-27T07:37:39,797 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=edb651d448a6733589b5e22423583e86, ASSIGN 2024-11-27T07:37:39,799 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=edb651d448a6733589b5e22423583e86, ASSIGN; state=OFFLINE, location=289954c9202d,41527,1732693002367; forceNewPlan=false, retain=false 2024-11-27T07:37:39,951 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=edb651d448a6733589b5e22423583e86, regionState=OPENING, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:39,958 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=edb651d448a6733589b5e22423583e86, ASSIGN because future has completed 2024-11-27T07:37:39,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure edb651d448a6733589b5e22423583e86, server=289954c9202d,41527,1732693002367}] 2024-11-27T07:37:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-27T07:37:40,125 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. 2024-11-27T07:37:40,126 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => edb651d448a6733589b5e22423583e86, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:37:40,126 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner edb651d448a6733589b5e22423583e86 2024-11-27T07:37:40,126 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:40,126 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for edb651d448a6733589b5e22423583e86 2024-11-27T07:37:40,127 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for edb651d448a6733589b5e22423583e86 2024-11-27T07:37:40,130 INFO [StoreOpener-edb651d448a6733589b5e22423583e86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region edb651d448a6733589b5e22423583e86 2024-11-27T07:37:40,132 INFO [StoreOpener-edb651d448a6733589b5e22423583e86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edb651d448a6733589b5e22423583e86 columnFamilyName testFamily 2024-11-27T07:37:40,132 DEBUG [StoreOpener-edb651d448a6733589b5e22423583e86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:40,133 INFO [StoreOpener-edb651d448a6733589b5e22423583e86-1 {}] regionserver.HStore(327): Store=edb651d448a6733589b5e22423583e86/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:37:40,133 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for edb651d448a6733589b5e22423583e86 2024-11-27T07:37:40,135 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScanner/edb651d448a6733589b5e22423583e86 2024-11-27T07:37:40,135 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScanner/edb651d448a6733589b5e22423583e86 2024-11-27T07:37:40,136 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for edb651d448a6733589b5e22423583e86 2024-11-27T07:37:40,136 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for edb651d448a6733589b5e22423583e86 2024-11-27T07:37:40,139 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for edb651d448a6733589b5e22423583e86 2024-11-27T07:37:40,143 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScanner/edb651d448a6733589b5e22423583e86/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:37:40,143 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened edb651d448a6733589b5e22423583e86; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75063007, jitterRate=0.11852596700191498}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T07:37:40,144 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edb651d448a6733589b5e22423583e86 2024-11-27T07:37:40,144 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for edb651d448a6733589b5e22423583e86: Running coprocessor pre-open hook at 1732693060127Writing region info on filesystem at 1732693060127Initializing all the Stores at 1732693060129 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693060129Cleaning up temporary data from old regions at 1732693060136 (+7 ms)Running coprocessor post-open hooks at 1732693060144 (+8 ms)Region opened successfully at 1732693060144 2024-11-27T07:37:40,146 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., pid=18, masterSystemTime=1732693060115 2024-11-27T07:37:40,148 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. 2024-11-27T07:37:40,149 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. 2024-11-27T07:37:40,149 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=edb651d448a6733589b5e22423583e86, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:40,152 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure edb651d448a6733589b5e22423583e86, server=289954c9202d,41527,1732693002367 because future has completed 2024-11-27T07:37:40,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-27T07:37:40,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure edb651d448a6733589b5e22423583e86, server=289954c9202d,41527,1732693002367 in 195 msec 2024-11-27T07:37:40,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-27T07:37:40,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=edb651d448a6733589b5e22423583e86, ASSIGN in 361 msec 2024-11-27T07:37:40,161 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:37:40,162 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693060162"}]},"ts":"1732693060162"} 2024-11-27T07:37:40,164 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-27T07:37:40,166 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:37:40,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2230 sec 2024-11-27T07:37:40,298 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T07:37:41,550 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-27T07:37:41,550 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-27T07:37:41,553 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-27T07:37:41,553 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-27T07:37:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-27T07:37:41,983 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-27T07:37:41,983 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-27T07:37:41,983 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:37:41,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-27T07:37:41,994 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:37:41,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-27T07:37:41,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., hostname=289954c9202d,41527,1732693002367, seqNum=2] 2024-11-27T07:37:42,012 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T07:37:42,012 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-27T07:37:42,016 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-27T07:37:42,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-27T07:37:43,025 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., hostname=289954c9202d,41527,1732693002367, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., hostname=289954c9202d,41527,1732693002367, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=289954c9202d:41527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-27T07:37:43,026 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., hostname=289954c9202d,41527,1732693002367, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-27T07:37:43,026 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., hostname=289954c9202d,41527,1732693002367, seqNum=2 from cache 2024-11-27T07:37:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-27T07:37:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., hostname=289954c9202d,41527,1732693002367, seqNum=2] 2024-11-27T07:37:44,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-27T07:37:44,141 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:37:45,025 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., hostname=289954c9202d,41527,1732693002367, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., hostname=289954c9202d,41527,1732693002367, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=289954c9202d:41527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=979ms, rpcTimeout=970ms 2024-11-27T07:37:45,025 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., hostname=289954c9202d,41527,1732693002367, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=979ms, rpcTimeout=970ms 2024-11-27T07:37:45,025 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., hostname=289954c9202d,41527,1732693002367, seqNum=2 from cache 2024-11-27T07:37:45,038 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 226), OpenFileDescriptor=448 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=202 (was 220), ProcessCount=11 (was 11), AvailableMemoryMB=5396 (was 5408) 2024-11-27T07:37:45,049 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=202, ProcessCount=11, AvailableMemoryMB=5396 2024-11-27T07:37:45,053 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:37:45,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-27T07:37:45,056 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:37:45,057 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:45,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-27T07:37:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T07:37:45,058 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:37:45,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741845_1021 (size=416) 2024-11-27T07:37:45,472 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 683f368212c8f5a82c099782900ab05b, NAME => 'testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c 2024-11-27T07:37:45,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741846_1022 (size=67) 2024-11-27T07:37:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-27T07:37:45,649 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-27T07:37:45,884 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:45,884 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 683f368212c8f5a82c099782900ab05b, disabling compactions & flushes 2024-11-27T07:37:45,885 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. 2024-11-27T07:37:45,885 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. 2024-11-27T07:37:45,885 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. after waiting 0 ms 2024-11-27T07:37:45,885 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. 2024-11-27T07:37:45,885 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. 2024-11-27T07:37:45,885 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 683f368212c8f5a82c099782900ab05b: Waiting for close lock at 1732693065884Disabling compacts and flushes for region at 1732693065884Disabling writes for close at 1732693065885 (+1 ms)Writing region close event to WAL at 1732693065885Closed at 1732693065885 2024-11-27T07:37:45,888 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:37:45,888 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732693065888"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732693065888"}]},"ts":"1732693065888"} 2024-11-27T07:37:45,892 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:37:45,894 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:37:45,894 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693065894"}]},"ts":"1732693065894"} 2024-11-27T07:37:45,897 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-27T07:37:45,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=683f368212c8f5a82c099782900ab05b, ASSIGN}] 2024-11-27T07:37:45,900 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=683f368212c8f5a82c099782900ab05b, ASSIGN 2024-11-27T07:37:45,901 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=683f368212c8f5a82c099782900ab05b, ASSIGN; state=OFFLINE, location=289954c9202d,41527,1732693002367; forceNewPlan=false, retain=false 2024-11-27T07:37:46,053 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=683f368212c8f5a82c099782900ab05b, regionState=OPENING, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:46,060 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=683f368212c8f5a82c099782900ab05b, ASSIGN because future has completed 2024-11-27T07:37:46,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 683f368212c8f5a82c099782900ab05b, server=289954c9202d,41527,1732693002367}] 2024-11-27T07:37:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T07:37:46,229 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. 2024-11-27T07:37:46,229 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 683f368212c8f5a82c099782900ab05b, NAME => 'testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:37:46,230 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 683f368212c8f5a82c099782900ab05b 2024-11-27T07:37:46,230 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:46,230 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 683f368212c8f5a82c099782900ab05b 2024-11-27T07:37:46,230 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 683f368212c8f5a82c099782900ab05b 2024-11-27T07:37:46,232 INFO [StoreOpener-683f368212c8f5a82c099782900ab05b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 683f368212c8f5a82c099782900ab05b 2024-11-27T07:37:46,235 INFO [StoreOpener-683f368212c8f5a82c099782900ab05b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 683f368212c8f5a82c099782900ab05b columnFamilyName testFamily 2024-11-27T07:37:46,235 DEBUG [StoreOpener-683f368212c8f5a82c099782900ab05b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:46,235 INFO [StoreOpener-683f368212c8f5a82c099782900ab05b-1 {}] regionserver.HStore(327): Store=683f368212c8f5a82c099782900ab05b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:37:46,236 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 683f368212c8f5a82c099782900ab05b 2024-11-27T07:37:46,237 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNextAsync/683f368212c8f5a82c099782900ab05b 2024-11-27T07:37:46,238 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNextAsync/683f368212c8f5a82c099782900ab05b 2024-11-27T07:37:46,238 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 683f368212c8f5a82c099782900ab05b 2024-11-27T07:37:46,238 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 683f368212c8f5a82c099782900ab05b 2024-11-27T07:37:46,241 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 683f368212c8f5a82c099782900ab05b 2024-11-27T07:37:46,244 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNextAsync/683f368212c8f5a82c099782900ab05b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:37:46,245 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 683f368212c8f5a82c099782900ab05b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59489958, jitterRate=-0.11353054642677307}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T07:37:46,245 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 683f368212c8f5a82c099782900ab05b 2024-11-27T07:37:46,245 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 683f368212c8f5a82c099782900ab05b: Running coprocessor pre-open hook at 1732693066230Writing region info on filesystem at 1732693066230Initializing all the Stores at 1732693066232 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693066232Cleaning up temporary data from old regions at 1732693066238 (+6 ms)Running coprocessor post-open hooks at 1732693066245 (+7 ms)Region opened successfully at 1732693066245 2024-11-27T07:37:46,246 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b., pid=21, masterSystemTime=1732693066215 2024-11-27T07:37:46,249 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. 2024-11-27T07:37:46,249 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. 2024-11-27T07:37:46,250 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=683f368212c8f5a82c099782900ab05b, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:46,253 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 683f368212c8f5a82c099782900ab05b, server=289954c9202d,41527,1732693002367 because future has completed 2024-11-27T07:37:46,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-27T07:37:46,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 683f368212c8f5a82c099782900ab05b, server=289954c9202d,41527,1732693002367 in 193 msec 2024-11-27T07:37:46,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-27T07:37:46,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=683f368212c8f5a82c099782900ab05b, ASSIGN in 360 msec 2024-11-27T07:37:46,261 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:37:46,262 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693066262"}]},"ts":"1732693066262"} 2024-11-27T07:37:46,264 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-27T07:37:46,265 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:37:46,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2120 sec 2024-11-27T07:37:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T07:37:48,083 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-27T07:37:48,083 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-27T07:37:48,084 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:37:48,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-27T07:37:48,095 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:37:48,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-27T07:37:48,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b., hostname=289954c9202d,41527,1732693002367, seqNum=2] 2024-11-27T07:37:48,112 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T07:37:48,112 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-27T07:37:48,116 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-27T07:37:48,118 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-27T07:37:48,120 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-27T07:37:48,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-27T07:37:51,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41527 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4569127999240142808' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-27T07:37:51,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41527 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:34646 deadline: 1732693072134, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4569127999240142808' does not exist 2024-11-27T07:37:51,136 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 289954c9202d,41527,1732693002367 for closing scanner id = -4569127999240142808 for 683f368212c8f5a82c099782900ab05b of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4569127999240142808' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:37:51,142 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=446 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=194 (was 202), ProcessCount=11 (was 11), AvailableMemoryMB=5399 (was 5396) - AvailableMemoryMB LEAK? - 2024-11-27T07:37:51,151 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=194, ProcessCount=11, AvailableMemoryMB=5398 2024-11-27T07:37:51,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:37:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-27T07:37:51,159 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:37:51,159 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:51,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-27T07:37:51,160 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:37:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-27T07:37:51,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741847_1023 (size=411) 2024-11-27T07:37:51,577 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c43ecc1a0eee0e96274e94265f82f2fe, NAME => 'testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c 2024-11-27T07:37:51,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741848_1024 (size=62) 2024-11-27T07:37:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-27T07:37:51,734 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:37:51,762 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-27T07:37:51,762 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T07:37:51,763 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-27T07:37:51,993 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:51,994 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing c43ecc1a0eee0e96274e94265f82f2fe, disabling compactions & flushes 2024-11-27T07:37:51,994 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. 2024-11-27T07:37:51,995 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. 2024-11-27T07:37:51,995 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. after waiting 0 ms 2024-11-27T07:37:51,995 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. 2024-11-27T07:37:51,995 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. 2024-11-27T07:37:51,996 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for c43ecc1a0eee0e96274e94265f82f2fe: Waiting for close lock at 1732693071994Disabling compacts and flushes for region at 1732693071994Disabling writes for close at 1732693071995 (+1 ms)Writing region close event to WAL at 1732693071995Closed at 1732693071995 2024-11-27T07:37:51,998 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:37:51,998 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732693071998"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732693071998"}]},"ts":"1732693071998"} 2024-11-27T07:37:52,001 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:37:52,002 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:37:52,003 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693072003"}]},"ts":"1732693072003"} 2024-11-27T07:37:52,005 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-27T07:37:52,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=c43ecc1a0eee0e96274e94265f82f2fe, ASSIGN}] 2024-11-27T07:37:52,008 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=c43ecc1a0eee0e96274e94265f82f2fe, ASSIGN 2024-11-27T07:37:52,009 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=c43ecc1a0eee0e96274e94265f82f2fe, ASSIGN; state=OFFLINE, location=289954c9202d,41527,1732693002367; forceNewPlan=false, retain=false 2024-11-27T07:37:52,160 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=c43ecc1a0eee0e96274e94265f82f2fe, regionState=OPENING, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:52,169 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=c43ecc1a0eee0e96274e94265f82f2fe, ASSIGN because future has completed 2024-11-27T07:37:52,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure c43ecc1a0eee0e96274e94265f82f2fe, server=289954c9202d,41527,1732693002367}] 2024-11-27T07:37:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-27T07:37:52,334 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. 2024-11-27T07:37:52,334 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => c43ecc1a0eee0e96274e94265f82f2fe, NAME => 'testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:37:52,334 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext c43ecc1a0eee0e96274e94265f82f2fe 2024-11-27T07:37:52,334 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:52,335 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for c43ecc1a0eee0e96274e94265f82f2fe 2024-11-27T07:37:52,335 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for c43ecc1a0eee0e96274e94265f82f2fe 2024-11-27T07:37:52,337 INFO [StoreOpener-c43ecc1a0eee0e96274e94265f82f2fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c43ecc1a0eee0e96274e94265f82f2fe 2024-11-27T07:37:52,339 INFO [StoreOpener-c43ecc1a0eee0e96274e94265f82f2fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c43ecc1a0eee0e96274e94265f82f2fe columnFamilyName testFamily 2024-11-27T07:37:52,339 DEBUG [StoreOpener-c43ecc1a0eee0e96274e94265f82f2fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:52,339 INFO [StoreOpener-c43ecc1a0eee0e96274e94265f82f2fe-1 {}] regionserver.HStore(327): Store=c43ecc1a0eee0e96274e94265f82f2fe/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:37:52,340 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for c43ecc1a0eee0e96274e94265f82f2fe 2024-11-27T07:37:52,341 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNext/c43ecc1a0eee0e96274e94265f82f2fe 2024-11-27T07:37:52,341 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNext/c43ecc1a0eee0e96274e94265f82f2fe 2024-11-27T07:37:52,342 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for c43ecc1a0eee0e96274e94265f82f2fe 2024-11-27T07:37:52,342 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for c43ecc1a0eee0e96274e94265f82f2fe 2024-11-27T07:37:52,345 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for c43ecc1a0eee0e96274e94265f82f2fe 2024-11-27T07:37:52,349 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNext/c43ecc1a0eee0e96274e94265f82f2fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:37:52,350 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened c43ecc1a0eee0e96274e94265f82f2fe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71638370, jitterRate=0.06749489903450012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T07:37:52,350 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c43ecc1a0eee0e96274e94265f82f2fe 2024-11-27T07:37:52,351 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for c43ecc1a0eee0e96274e94265f82f2fe: Running coprocessor pre-open hook at 1732693072335Writing region info on filesystem at 1732693072335Initializing all the Stores at 1732693072336 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693072336Cleaning up temporary data from old regions at 1732693072342 (+6 ms)Running coprocessor post-open hooks at 1732693072350 (+8 ms)Region opened successfully at 1732693072351 (+1 ms) 2024-11-27T07:37:52,353 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe., pid=24, masterSystemTime=1732693072324 2024-11-27T07:37:52,356 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. 2024-11-27T07:37:52,356 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. 2024-11-27T07:37:52,357 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=c43ecc1a0eee0e96274e94265f82f2fe, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:52,359 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure c43ecc1a0eee0e96274e94265f82f2fe, server=289954c9202d,41527,1732693002367 because future has completed 2024-11-27T07:37:52,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-27T07:37:52,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure c43ecc1a0eee0e96274e94265f82f2fe, server=289954c9202d,41527,1732693002367 in 192 msec 2024-11-27T07:37:52,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-27T07:37:52,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=c43ecc1a0eee0e96274e94265f82f2fe, ASSIGN in 358 msec 2024-11-27T07:37:52,368 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:37:52,368 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693072368"}]},"ts":"1732693072368"} 2024-11-27T07:37:52,371 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-27T07:37:52,372 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:37:52,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2180 sec 2024-11-27T07:37:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-27T07:37:54,213 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-27T07:37:54,213 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-27T07:37:54,214 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:37:54,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-27T07:37:54,223 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:37:54,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-27T07:37:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe., hostname=289954c9202d,41527,1732693002367, seqNum=2] 2024-11-27T07:37:54,238 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T07:37:54,238 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-27T07:37:54,242 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-27T07:37:54,245 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-27T07:37:54,246 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-27T07:37:54,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-27T07:37:56,372 INFO [master/289954c9202d:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-27T07:37:56,372 INFO [master/289954c9202d:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-27T07:37:57,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41527 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4569127999240142804' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-27T07:37:57,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41527 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:34646 deadline: 1732693078264, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4569127999240142804' does not exist 2024-11-27T07:37:57,267 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 289954c9202d,41527,1732693002367 for closing scanner id = -4569127999240142804 for c43ecc1a0eee0e96274e94265f82f2fe of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4569127999240142804' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:37:57,274 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=164 (was 194), ProcessCount=11 (was 11), AvailableMemoryMB=5396 (was 5398) 2024-11-27T07:37:57,283 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=164, ProcessCount=11, AvailableMemoryMB=5395 2024-11-27T07:37:57,285 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:37:57,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-27T07:37:57,289 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:37:57,289 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:57,289 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-27T07:37:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T07:37:57,291 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:37:57,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741849_1025 (size=419) 2024-11-27T07:37:57,708 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dad01c63aab200f54826187889d7bb3a, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c 2024-11-27T07:37:57,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741850_1026 (size=70) 2024-11-27T07:37:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-27T07:37:57,839 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:37:57,871 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-27T07:37:58,123 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:58,123 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing dad01c63aab200f54826187889d7bb3a, disabling compactions & flushes 2024-11-27T07:37:58,123 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. 2024-11-27T07:37:58,124 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. 2024-11-27T07:37:58,124 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. after waiting 0 ms 2024-11-27T07:37:58,124 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. 2024-11-27T07:37:58,124 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. 2024-11-27T07:37:58,124 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for dad01c63aab200f54826187889d7bb3a: Waiting for close lock at 1732693078123Disabling compacts and flushes for region at 1732693078123Disabling writes for close at 1732693078124 (+1 ms)Writing region close event to WAL at 1732693078124Closed at 1732693078124 2024-11-27T07:37:58,129 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:37:58,130 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732693078129"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732693078129"}]},"ts":"1732693078129"} 2024-11-27T07:37:58,133 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:37:58,134 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:37:58,134 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693078134"}]},"ts":"1732693078134"} 2024-11-27T07:37:58,137 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-27T07:37:58,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=dad01c63aab200f54826187889d7bb3a, ASSIGN}] 2024-11-27T07:37:58,138 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=dad01c63aab200f54826187889d7bb3a, ASSIGN 2024-11-27T07:37:58,139 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=dad01c63aab200f54826187889d7bb3a, ASSIGN; state=OFFLINE, location=289954c9202d,41527,1732693002367; forceNewPlan=false, retain=false 2024-11-27T07:37:58,291 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=dad01c63aab200f54826187889d7bb3a, regionState=OPENING, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:58,295 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=dad01c63aab200f54826187889d7bb3a, ASSIGN because future has completed 2024-11-27T07:37:58,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure dad01c63aab200f54826187889d7bb3a, server=289954c9202d,41527,1732693002367}] 2024-11-27T07:37:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T07:37:58,463 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. 2024-11-27T07:37:58,464 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => dad01c63aab200f54826187889d7bb3a, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:37:58,464 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync dad01c63aab200f54826187889d7bb3a 2024-11-27T07:37:58,464 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:37:58,465 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for dad01c63aab200f54826187889d7bb3a 2024-11-27T07:37:58,465 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for dad01c63aab200f54826187889d7bb3a 2024-11-27T07:37:58,468 INFO [StoreOpener-dad01c63aab200f54826187889d7bb3a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region dad01c63aab200f54826187889d7bb3a 2024-11-27T07:37:58,471 INFO [StoreOpener-dad01c63aab200f54826187889d7bb3a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dad01c63aab200f54826187889d7bb3a columnFamilyName testFamily 2024-11-27T07:37:58,472 DEBUG [StoreOpener-dad01c63aab200f54826187889d7bb3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:37:58,472 INFO [StoreOpener-dad01c63aab200f54826187889d7bb3a-1 {}] regionserver.HStore(327): Store=dad01c63aab200f54826187889d7bb3a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:37:58,472 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for dad01c63aab200f54826187889d7bb3a 2024-11-27T07:37:58,474 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/dad01c63aab200f54826187889d7bb3a 2024-11-27T07:37:58,474 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/dad01c63aab200f54826187889d7bb3a 2024-11-27T07:37:58,475 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for dad01c63aab200f54826187889d7bb3a 2024-11-27T07:37:58,475 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for dad01c63aab200f54826187889d7bb3a 2024-11-27T07:37:58,478 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for dad01c63aab200f54826187889d7bb3a 2024-11-27T07:37:58,481 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/dad01c63aab200f54826187889d7bb3a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:37:58,481 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened dad01c63aab200f54826187889d7bb3a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66553829, jitterRate=-0.008270666003227234}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T07:37:58,482 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dad01c63aab200f54826187889d7bb3a 2024-11-27T07:37:58,482 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for dad01c63aab200f54826187889d7bb3a: Running coprocessor pre-open hook at 1732693078465Writing region info on filesystem at 1732693078465Initializing all the Stores at 1732693078468 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693078468Cleaning up temporary data from old regions at 1732693078475 (+7 ms)Running coprocessor post-open hooks at 1732693078482 (+7 ms)Region opened successfully at 1732693078482 2024-11-27T07:37:58,484 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a., pid=27, masterSystemTime=1732693078449 2024-11-27T07:37:58,486 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. 2024-11-27T07:37:58,487 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. 2024-11-27T07:37:58,487 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=dad01c63aab200f54826187889d7bb3a, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:37:58,490 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure dad01c63aab200f54826187889d7bb3a, server=289954c9202d,41527,1732693002367 because future has completed 2024-11-27T07:37:58,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-27T07:37:58,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure dad01c63aab200f54826187889d7bb3a, server=289954c9202d,41527,1732693002367 in 195 msec 2024-11-27T07:37:58,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-27T07:37:58,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=dad01c63aab200f54826187889d7bb3a, ASSIGN in 357 msec 2024-11-27T07:37:58,498 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:37:58,498 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693078498"}]},"ts":"1732693078498"} 2024-11-27T07:37:58,501 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-27T07:37:58,502 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:37:58,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2170 sec 2024-11-27T07:38:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T07:38:00,322 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-27T07:38:00,322 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-27T07:38:00,323 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:38:00,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-27T07:38:00,333 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:38:00,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-27T07:38:00,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a., hostname=289954c9202d,41527,1732693002367, seqNum=2] 2024-11-27T07:38:00,347 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T07:38:00,347 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-27T07:38:00,351 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-27T07:38:00,353 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-27T07:38:01,550 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-27T07:38:01,550 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-27T07:38:01,552 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-27T07:38:01,552 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T07:38:03,972 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:38:04,004 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-27T07:38:06,355 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a., hostname=289954c9202d,41527,1732693002367, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a., hostname=289954c9202d,41527,1732693002367, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=289954c9202d:41527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6002ms, rpcTimeout=5999ms 2024-11-27T07:38:06,355 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a., hostname=289954c9202d,41527,1732693002367, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6002ms, rpcTimeout=5999ms 2024-11-27T07:38:06,356 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a., hostname=289954c9202d,41527,1732693002367, seqNum=2 from cache 2024-11-27T07:38:06,377 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=159 (was 164), ProcessCount=11 (was 11), AvailableMemoryMB=5394 (was 5395) 2024-11-27T07:38:06,386 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=159, ProcessCount=11, AvailableMemoryMB=5393 2024-11-27T07:38:06,389 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:38:06,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-27T07:38:06,392 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:38:06,392 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:38:06,393 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-27T07:38:06,394 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:38:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-27T07:38:06,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741851_1027 (size=407) 2024-11-27T07:38:06,811 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a5fb9e44e8217707139da648e2bcde24, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c 2024-11-27T07:38:06,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741852_1028 (size=58) 2024-11-27T07:38:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-27T07:38:07,226 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:38:07,226 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing a5fb9e44e8217707139da648e2bcde24, disabling compactions & flushes 2024-11-27T07:38:07,227 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. 2024-11-27T07:38:07,227 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. 2024-11-27T07:38:07,227 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. after waiting 0 ms 2024-11-27T07:38:07,227 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. 2024-11-27T07:38:07,227 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. 2024-11-27T07:38:07,228 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for a5fb9e44e8217707139da648e2bcde24: Waiting for close lock at 1732693087226Disabling compacts and flushes for region at 1732693087226Disabling writes for close at 1732693087227 (+1 ms)Writing region close event to WAL at 1732693087227Closed at 1732693087227 2024-11-27T07:38:07,230 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:38:07,231 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732693087230"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732693087230"}]},"ts":"1732693087230"} 2024-11-27T07:38:07,235 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:38:07,238 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:38:07,238 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693087238"}]},"ts":"1732693087238"} 2024-11-27T07:38:07,242 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-27T07:38:07,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a5fb9e44e8217707139da648e2bcde24, ASSIGN}] 2024-11-27T07:38:07,244 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a5fb9e44e8217707139da648e2bcde24, ASSIGN 2024-11-27T07:38:07,246 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a5fb9e44e8217707139da648e2bcde24, ASSIGN; state=OFFLINE, location=289954c9202d,41527,1732693002367; forceNewPlan=false, retain=false 2024-11-27T07:38:07,397 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a5fb9e44e8217707139da648e2bcde24, regionState=OPENING, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:38:07,404 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a5fb9e44e8217707139da648e2bcde24, ASSIGN because future has completed 2024-11-27T07:38:07,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5fb9e44e8217707139da648e2bcde24, server=289954c9202d,41527,1732693002367}] 2024-11-27T07:38:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-27T07:38:07,572 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. 2024-11-27T07:38:07,572 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => a5fb9e44e8217707139da648e2bcde24, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:38:07,573 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext a5fb9e44e8217707139da648e2bcde24 2024-11-27T07:38:07,573 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:38:07,573 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for a5fb9e44e8217707139da648e2bcde24 2024-11-27T07:38:07,573 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for a5fb9e44e8217707139da648e2bcde24 2024-11-27T07:38:07,575 INFO [StoreOpener-a5fb9e44e8217707139da648e2bcde24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a5fb9e44e8217707139da648e2bcde24 2024-11-27T07:38:07,577 INFO [StoreOpener-a5fb9e44e8217707139da648e2bcde24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5fb9e44e8217707139da648e2bcde24 columnFamilyName testFamily 2024-11-27T07:38:07,577 DEBUG [StoreOpener-a5fb9e44e8217707139da648e2bcde24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:38:07,578 INFO [StoreOpener-a5fb9e44e8217707139da648e2bcde24-1 {}] regionserver.HStore(327): Store=a5fb9e44e8217707139da648e2bcde24/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:38:07,578 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for a5fb9e44e8217707139da648e2bcde24 2024-11-27T07:38:07,579 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNext/a5fb9e44e8217707139da648e2bcde24 2024-11-27T07:38:07,580 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNext/a5fb9e44e8217707139da648e2bcde24 2024-11-27T07:38:07,580 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for a5fb9e44e8217707139da648e2bcde24 2024-11-27T07:38:07,580 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for a5fb9e44e8217707139da648e2bcde24 2024-11-27T07:38:07,583 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for a5fb9e44e8217707139da648e2bcde24 2024-11-27T07:38:07,586 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNext/a5fb9e44e8217707139da648e2bcde24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:38:07,587 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened a5fb9e44e8217707139da648e2bcde24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63477227, jitterRate=-0.05411560833454132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T07:38:07,587 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5fb9e44e8217707139da648e2bcde24 2024-11-27T07:38:07,588 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for a5fb9e44e8217707139da648e2bcde24: Running coprocessor pre-open hook at 1732693087573Writing region info on filesystem at 1732693087573Initializing all the Stores at 1732693087575 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693087575Cleaning up temporary data from old regions at 1732693087580 (+5 ms)Running coprocessor post-open hooks at 1732693087587 (+7 ms)Region opened successfully at 1732693087588 (+1 ms) 2024-11-27T07:38:07,589 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24., pid=30, masterSystemTime=1732693087559 2024-11-27T07:38:07,592 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. 2024-11-27T07:38:07,592 INFO [RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. 2024-11-27T07:38:07,593 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a5fb9e44e8217707139da648e2bcde24, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:38:07,596 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5fb9e44e8217707139da648e2bcde24, server=289954c9202d,41527,1732693002367 because future has completed 2024-11-27T07:38:07,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-27T07:38:07,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure a5fb9e44e8217707139da648e2bcde24, server=289954c9202d,41527,1732693002367 in 193 msec 2024-11-27T07:38:07,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-27T07:38:07,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a5fb9e44e8217707139da648e2bcde24, ASSIGN in 357 msec 2024-11-27T07:38:07,604 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:38:07,604 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693087604"}]},"ts":"1732693087604"} 2024-11-27T07:38:07,607 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-27T07:38:07,608 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:38:07,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2190 sec 2024-11-27T07:38:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-27T07:38:09,433 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-27T07:38:09,433 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-27T07:38:09,433 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:38:09,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-27T07:38:09,445 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:38:09,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-27T07:38:09,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24., hostname=289954c9202d,41527,1732693002367, seqNum=2] 2024-11-27T07:38:09,459 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T07:38:09,460 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-27T07:38:09,463 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-27T07:38:09,465 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-27T07:38:09,467 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-27T07:38:09,467 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-27T07:38:10,007 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-27T07:38:10,299 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T07:38:11,550 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-27T07:38:11,551 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-27T07:38:13,079 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:38:15,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41527 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4569127999240142796' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-27T07:38:15,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41527 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:34646 deadline: 1732693101476, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4569127999240142796' does not exist 2024-11-27T07:38:15,480 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 289954c9202d,41527,1732693002367 for closing scanner id = -4569127999240142796 for a5fb9e44e8217707139da648e2bcde24 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-4569127999240142796' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:38:15,485 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=134 (was 159), ProcessCount=11 (was 11), AvailableMemoryMB=5394 (was 5393) - AvailableMemoryMB LEAK? - 2024-11-27T07:38:15,493 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=134, ProcessCount=11, AvailableMemoryMB=5394 2024-11-27T07:38:15,495 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:38:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-27T07:38:15,498 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:38:15,498 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:38:15,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-27T07:38:15,500 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:38:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T07:38:15,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741853_1029 (size=423) 2024-11-27T07:38:15,917 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 65aaf181dbb3f882270fa6339a32aa32, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c 2024-11-27T07:38:15,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741854_1030 (size=74) 2024-11-27T07:38:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-27T07:38:16,331 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:38:16,331 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 65aaf181dbb3f882270fa6339a32aa32, disabling compactions & flushes 2024-11-27T07:38:16,331 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. 2024-11-27T07:38:16,331 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. 2024-11-27T07:38:16,332 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. after waiting 0 ms 2024-11-27T07:38:16,332 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. 2024-11-27T07:38:16,332 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. 2024-11-27T07:38:16,332 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 65aaf181dbb3f882270fa6339a32aa32: Waiting for close lock at 1732693096331Disabling compacts and flushes for region at 1732693096331Disabling writes for close at 1732693096332 (+1 ms)Writing region close event to WAL at 1732693096332Closed at 1732693096332 2024-11-27T07:38:16,335 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:38:16,336 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732693096335"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732693096335"}]},"ts":"1732693096335"} 2024-11-27T07:38:16,340 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:38:16,342 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:38:16,342 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693096342"}]},"ts":"1732693096342"} 2024-11-27T07:38:16,344 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-27T07:38:16,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=65aaf181dbb3f882270fa6339a32aa32, ASSIGN}] 2024-11-27T07:38:16,346 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=65aaf181dbb3f882270fa6339a32aa32, ASSIGN 2024-11-27T07:38:16,347 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=65aaf181dbb3f882270fa6339a32aa32, ASSIGN; state=OFFLINE, location=289954c9202d,41527,1732693002367; forceNewPlan=false, retain=false 2024-11-27T07:38:16,470 DEBUG [master/289954c9202d:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-27T07:38:16,497 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=65aaf181dbb3f882270fa6339a32aa32, regionState=OPENING, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:38:16,501 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=65aaf181dbb3f882270fa6339a32aa32, ASSIGN because future has completed 2024-11-27T07:38:16,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65aaf181dbb3f882270fa6339a32aa32, server=289954c9202d,41527,1732693002367}] 2024-11-27T07:38:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T07:38:16,663 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. 2024-11-27T07:38:16,663 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 65aaf181dbb3f882270fa6339a32aa32, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:38:16,664 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 65aaf181dbb3f882270fa6339a32aa32 2024-11-27T07:38:16,665 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:38:16,665 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 65aaf181dbb3f882270fa6339a32aa32 2024-11-27T07:38:16,665 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 65aaf181dbb3f882270fa6339a32aa32 2024-11-27T07:38:16,668 INFO [StoreOpener-65aaf181dbb3f882270fa6339a32aa32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 65aaf181dbb3f882270fa6339a32aa32 2024-11-27T07:38:16,670 INFO [StoreOpener-65aaf181dbb3f882270fa6339a32aa32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65aaf181dbb3f882270fa6339a32aa32 columnFamilyName testFamily 2024-11-27T07:38:16,671 DEBUG [StoreOpener-65aaf181dbb3f882270fa6339a32aa32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:38:16,671 INFO [StoreOpener-65aaf181dbb3f882270fa6339a32aa32-1 {}] regionserver.HStore(327): Store=65aaf181dbb3f882270fa6339a32aa32/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:38:16,672 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 65aaf181dbb3f882270fa6339a32aa32 2024-11-27T07:38:16,673 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScannerAsync/65aaf181dbb3f882270fa6339a32aa32 2024-11-27T07:38:16,674 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScannerAsync/65aaf181dbb3f882270fa6339a32aa32 2024-11-27T07:38:16,675 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 65aaf181dbb3f882270fa6339a32aa32 2024-11-27T07:38:16,675 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 65aaf181dbb3f882270fa6339a32aa32 2024-11-27T07:38:16,678 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 65aaf181dbb3f882270fa6339a32aa32 2024-11-27T07:38:16,681 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScannerAsync/65aaf181dbb3f882270fa6339a32aa32/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:38:16,682 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 65aaf181dbb3f882270fa6339a32aa32; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74211404, jitterRate=0.10583609342575073}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T07:38:16,682 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 65aaf181dbb3f882270fa6339a32aa32 2024-11-27T07:38:16,683 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 65aaf181dbb3f882270fa6339a32aa32: Running coprocessor pre-open hook at 1732693096665Writing region info on filesystem at 1732693096665Initializing all the Stores at 1732693096668 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732693096668Cleaning up temporary data from old regions at 1732693096675 (+7 ms)Running coprocessor post-open hooks at 1732693096682 (+7 ms)Region opened successfully at 1732693096683 (+1 ms) 2024-11-27T07:38:16,685 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32., pid=33, masterSystemTime=1732693096655 2024-11-27T07:38:16,687 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. 2024-11-27T07:38:16,688 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. 2024-11-27T07:38:16,688 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=65aaf181dbb3f882270fa6339a32aa32, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,41527,1732693002367 2024-11-27T07:38:16,691 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65aaf181dbb3f882270fa6339a32aa32, server=289954c9202d,41527,1732693002367 because future has completed 2024-11-27T07:38:16,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-27T07:38:16,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 65aaf181dbb3f882270fa6339a32aa32, server=289954c9202d,41527,1732693002367 in 192 msec 2024-11-27T07:38:16,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-27T07:38:16,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=65aaf181dbb3f882270fa6339a32aa32, ASSIGN in 351 msec 2024-11-27T07:38:16,699 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:38:16,699 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732693096699"}]},"ts":"1732693096699"} 2024-11-27T07:38:16,701 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-27T07:38:16,702 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:38:16,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2070 sec 2024-11-27T07:38:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T07:38:18,522 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-27T07:38:18,522 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-27T07:38:18,523 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:38:18,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-27T07:38:18,533 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T07:38:18,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-27T07:38:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32., hostname=289954c9202d,41527,1732693002367, seqNum=2] 2024-11-27T07:38:18,547 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T07:38:18,547 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-27T07:38:18,551 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-27T07:38:18,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-27T07:38:19,556 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32., hostname=289954c9202d,41527,1732693002367, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32., hostname=289954c9202d,41527,1732693002367, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=289954c9202d:41527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-27T07:38:19,556 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32., hostname=289954c9202d,41527,1732693002367, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-27T07:38:19,556 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32., hostname=289954c9202d,41527,1732693002367, seqNum=2 from cache 2024-11-27T07:38:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-27T07:38:20,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32., hostname=289954c9202d,41527,1732693002367, seqNum=2] 2024-11-27T07:38:20,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41527 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-27T07:38:21,550 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-27T07:38:21,550 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T07:38:21,555 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32., hostname=289954c9202d,41527,1732693002367, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32., hostname=289954c9202d,41527,1732693002367, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=289954c9202d:41527 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=978ms, rpcTimeout=974ms 2024-11-27T07:38:21,556 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32., hostname=289954c9202d,41527,1732693002367, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=978ms, rpcTimeout=974ms 2024-11-27T07:38:21,556 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32., hostname=289954c9202d,41527,1732693002367, seqNum=2 from cache 2024-11-27T07:38:21,575 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=131 (was 134), ProcessCount=11 (was 11), AvailableMemoryMB=5394 (was 5394) 2024-11-27T07:38:21,576 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T07:38:21,576 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T07:38:21,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:38:21,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:38:21,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:38:21,583 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T07:38:21,583 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T07:38:21,583 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T07:38:21,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:38:21,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:38:21,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:38:21,583 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T07:38:21,583 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=534022733, stopped=false 2024-11-27T07:38:21,584 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=289954c9202d,41501,1732693001213 2024-11-27T07:38:21,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T07:38:21,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T07:38:21,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:38:21,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:38:21,647 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T07:38:21,648 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T07:38:21,649 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T07:38:21,649 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T07:38:21,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:38:21,649 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T07:38:21,650 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '289954c9202d,41527,1732693002367' ***** 2024-11-27T07:38:21,650 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T07:38:21,650 INFO [RS:0;289954c9202d:41527 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T07:38:21,651 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T07:38:21,651 INFO [RS:0;289954c9202d:41527 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T07:38:21,651 INFO [RS:0;289954c9202d:41527 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T07:38:21,651 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(3091): Received CLOSE for a5fb9e44e8217707139da648e2bcde24 2024-11-27T07:38:21,652 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(3091): Received CLOSE for 282935bd9dd931fa45e861f636eef90f 2024-11-27T07:38:21,652 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(3091): Received CLOSE for 65aaf181dbb3f882270fa6339a32aa32 2024-11-27T07:38:21,652 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(3091): Received CLOSE for c43ecc1a0eee0e96274e94265f82f2fe 2024-11-27T07:38:21,652 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(3091): Received CLOSE for dad01c63aab200f54826187889d7bb3a 2024-11-27T07:38:21,652 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(3091): Received CLOSE for 0720db5474d7242cd64e0ebf2bd06ad8 2024-11-27T07:38:21,652 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(3091): Received CLOSE for 84e136ef3c363cd7fdbf7742a369e042 2024-11-27T07:38:21,653 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(3091): Received CLOSE for edb651d448a6733589b5e22423583e86 2024-11-27T07:38:21,653 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a5fb9e44e8217707139da648e2bcde24, disabling compactions & flushes 2024-11-27T07:38:21,653 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(3091): Received CLOSE for 683f368212c8f5a82c099782900ab05b 2024-11-27T07:38:21,653 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. 2024-11-27T07:38:21,653 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(3091): Received CLOSE for 2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:38:21,653 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. 2024-11-27T07:38:21,653 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(959): stopping server 289954c9202d,41527,1732693002367 2024-11-27T07:38:21,653 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. after waiting 0 ms 2024-11-27T07:38:21,653 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. 2024-11-27T07:38:21,653 INFO [RS:0;289954c9202d:41527 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T07:38:21,653 INFO [RS:0;289954c9202d:41527 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;289954c9202d:41527. 2024-11-27T07:38:21,653 DEBUG [RS:0;289954c9202d:41527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T07:38:21,653 DEBUG [RS:0;289954c9202d:41527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:38:21,654 INFO [RS:0;289954c9202d:41527 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T07:38:21,654 INFO [RS:0;289954c9202d:41527 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T07:38:21,654 INFO [RS:0;289954c9202d:41527 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T07:38:21,654 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T07:38:21,654 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-27T07:38:21,655 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T07:38:21,655 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T07:38:21,655 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T07:38:21,655 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T07:38:21,655 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 0720db5474d7242cd64e0ebf2bd06ad8, 1588230740, 282935bd9dd931fa45e861f636eef90f, 2c42380c364991ffc7b56aaffd602a09, 65aaf181dbb3f882270fa6339a32aa32, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, a5fb9e44e8217707139da648e2bcde24, c43ecc1a0eee0e96274e94265f82f2fe, dad01c63aab200f54826187889d7bb3a, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:21,655 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T07:38:21,658 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-27T07:38:21,658 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a5fb9e44e8217707139da648e2bcde24 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T07:38:21,708 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNext/a5fb9e44e8217707139da648e2bcde24/.tmp/testFamily/7af8b744ce574f418df0648b826ca00c is 61, key is row-0/testFamily:testQualifier/1732693089449/Put/seqid=0 2024-11-27T07:38:21,715 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/.tmp/info/bcbeebe8a52f495cb41481a89f68ccc4 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09./info:regioninfo/1732693037746/Put/seqid=0 2024-11-27T07:38:21,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741855_1031 (size=5049) 2024-11-27T07:38:21,718 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNext/a5fb9e44e8217707139da648e2bcde24/.tmp/testFamily/7af8b744ce574f418df0648b826ca00c 2024-11-27T07:38:21,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741856_1032 (size=20564) 2024-11-27T07:38:21,768 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNext/a5fb9e44e8217707139da648e2bcde24/.tmp/testFamily/7af8b744ce574f418df0648b826ca00c as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNext/a5fb9e44e8217707139da648e2bcde24/testFamily/7af8b744ce574f418df0648b826ca00c 2024-11-27T07:38:21,777 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNext/a5fb9e44e8217707139da648e2bcde24/testFamily/7af8b744ce574f418df0648b826ca00c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T07:38:21,785 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a5fb9e44e8217707139da648e2bcde24 in 126ms, sequenceid=8, compaction requested=false 2024-11-27T07:38:21,793 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNext/a5fb9e44e8217707139da648e2bcde24/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T07:38:21,795 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. 2024-11-27T07:38:21,795 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a5fb9e44e8217707139da648e2bcde24: Waiting for close lock at 1732693101652Running coprocessor pre-close hooks at 1732693101653 (+1 ms)Disabling compacts and flushes for region at 1732693101653Disabling writes for close at 1732693101653Obtaining lock to block concurrent updates at 1732693101658 (+5 ms)Preparing flush snapshotting stores in a5fb9e44e8217707139da648e2bcde24 at 1732693101658Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732693101666 (+8 ms)Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. at 1732693101667 (+1 ms)Flushing a5fb9e44e8217707139da648e2bcde24/testFamily: creating writer at 1732693101669 (+2 ms)Flushing a5fb9e44e8217707139da648e2bcde24/testFamily: appending metadata at 1732693101704 (+35 ms)Flushing a5fb9e44e8217707139da648e2bcde24/testFamily: closing flushed file at 1732693101706 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7116cfcc: reopening flushed file at 1732693101766 (+60 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a5fb9e44e8217707139da648e2bcde24 in 126ms, sequenceid=8, compaction requested=false at 1732693101786 (+20 ms)Writing region close event to WAL at 1732693101787 (+1 ms)Running coprocessor post-close hooks at 1732693101794 (+7 ms)Closed at 1732693101795 (+1 ms) 2024-11-27T07:38:21,796 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732693086388.a5fb9e44e8217707139da648e2bcde24. 2024-11-27T07:38:21,796 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 282935bd9dd931fa45e861f636eef90f, disabling compactions & flushes 2024-11-27T07:38:21,796 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. 2024-11-27T07:38:21,796 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. 2024-11-27T07:38:21,796 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. after waiting 0 ms 2024-11-27T07:38:21,797 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. 2024-11-27T07:38:21,797 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 282935bd9dd931fa45e861f636eef90f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T07:38:21,820 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextException/282935bd9dd931fa45e861f636eef90f/.tmp/testFamily/3d6c02e3127f4ed8a93bfd6e8c07c878 is 61, key is row-0/testFamily:testQualifier/1732693058888/Put/seqid=0 2024-11-27T07:38:21,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741857_1033 (size=5049) 2024-11-27T07:38:21,855 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 0720db5474d7242cd64e0ebf2bd06ad8, 1588230740, 282935bd9dd931fa45e861f636eef90f, 2c42380c364991ffc7b56aaffd602a09, 65aaf181dbb3f882270fa6339a32aa32, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, c43ecc1a0eee0e96274e94265f82f2fe, dad01c63aab200f54826187889d7bb3a, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:22,056 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 0720db5474d7242cd64e0ebf2bd06ad8, 1588230740, 282935bd9dd931fa45e861f636eef90f, 2c42380c364991ffc7b56aaffd602a09, 65aaf181dbb3f882270fa6339a32aa32, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, c43ecc1a0eee0e96274e94265f82f2fe, dad01c63aab200f54826187889d7bb3a, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:22,079 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-27T07:38:22,125 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/.tmp/info/bcbeebe8a52f495cb41481a89f68ccc4 2024-11-27T07:38:22,160 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/.tmp/ns/da22a1aaacca44b2b2d87e05200551de is 43, key is default/ns:d/1732693036216/Put/seqid=0 2024-11-27T07:38:22,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741858_1034 (size=5153) 2024-11-27T07:38:22,166 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:38:22,191 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-27T07:38:22,226 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextException/282935bd9dd931fa45e861f636eef90f/.tmp/testFamily/3d6c02e3127f4ed8a93bfd6e8c07c878 2024-11-27T07:38:22,235 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextException/282935bd9dd931fa45e861f636eef90f/.tmp/testFamily/3d6c02e3127f4ed8a93bfd6e8c07c878 as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextException/282935bd9dd931fa45e861f636eef90f/testFamily/3d6c02e3127f4ed8a93bfd6e8c07c878 2024-11-27T07:38:22,243 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextException/282935bd9dd931fa45e861f636eef90f/testFamily/3d6c02e3127f4ed8a93bfd6e8c07c878, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T07:38:22,244 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 282935bd9dd931fa45e861f636eef90f in 447ms, sequenceid=8, compaction requested=false 2024-11-27T07:38:22,250 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextException/282935bd9dd931fa45e861f636eef90f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T07:38:22,250 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. 2024-11-27T07:38:22,250 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 282935bd9dd931fa45e861f636eef90f: Waiting for close lock at 1732693101796Running coprocessor pre-close hooks at 1732693101796Disabling compacts and flushes for region at 1732693101796Disabling writes for close at 1732693101796Obtaining lock to block concurrent updates at 1732693101797 (+1 ms)Preparing flush snapshotting stores in 282935bd9dd931fa45e861f636eef90f at 1732693101797Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732693101797Flushing stores of testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. at 1732693101798 (+1 ms)Flushing 282935bd9dd931fa45e861f636eef90f/testFamily: creating writer at 1732693101798Flushing 282935bd9dd931fa45e861f636eef90f/testFamily: appending metadata at 1732693101819 (+21 ms)Flushing 282935bd9dd931fa45e861f636eef90f/testFamily: closing flushed file at 1732693101819Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58f4ab80: reopening flushed file at 1732693102233 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 282935bd9dd931fa45e861f636eef90f in 447ms, sequenceid=8, compaction requested=false at 1732693102244 (+11 ms)Writing region close event to WAL at 1732693102245 (+1 ms)Running coprocessor post-close hooks at 1732693102250 (+5 ms)Closed at 1732693102250 2024-11-27T07:38:22,251 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732693057862.282935bd9dd931fa45e861f636eef90f. 2024-11-27T07:38:22,251 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 65aaf181dbb3f882270fa6339a32aa32, disabling compactions & flushes 2024-11-27T07:38:22,251 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. 2024-11-27T07:38:22,251 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. 2024-11-27T07:38:22,251 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. after waiting 0 ms 2024-11-27T07:38:22,251 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. 2024-11-27T07:38:22,251 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 65aaf181dbb3f882270fa6339a32aa32 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T07:38:22,256 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 0720db5474d7242cd64e0ebf2bd06ad8, 1588230740, 2c42380c364991ffc7b56aaffd602a09, 65aaf181dbb3f882270fa6339a32aa32, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, c43ecc1a0eee0e96274e94265f82f2fe, dad01c63aab200f54826187889d7bb3a, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:22,267 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScannerAsync/65aaf181dbb3f882270fa6339a32aa32/.tmp/testFamily/7d2a92c8873a4d7f97e3ca732cea1fdb is 61, key is row-0/testFamily:testQualifier/1732693098538/Put/seqid=0 2024-11-27T07:38:22,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741859_1035 (size=5049) 2024-11-27T07:38:22,407 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=289954c9202d:41501 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 10 more 2024-11-27T07:38:22,456 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 0720db5474d7242cd64e0ebf2bd06ad8, 1588230740, 2c42380c364991ffc7b56aaffd602a09, 65aaf181dbb3f882270fa6339a32aa32, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, c43ecc1a0eee0e96274e94265f82f2fe, dad01c63aab200f54826187889d7bb3a, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:22,566 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/.tmp/ns/da22a1aaacca44b2b2d87e05200551de 2024-11-27T07:38:22,596 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/.tmp/table/dc705a4ddf4744f4ae1c7ca21a5ac060 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732693037762/Put/seqid=0 2024-11-27T07:38:22,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741860_1036 (size=6699) 2024-11-27T07:38:22,645 INFO [regionserver/289954c9202d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T07:38:22,645 INFO [regionserver/289954c9202d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T07:38:22,657 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-27T07:38:22,657 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1325): Online Regions={65aaf181dbb3f882270fa6339a32aa32=testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32., 1588230740=hbase:meta,,1.1588230740, c43ecc1a0eee0e96274e94265f82f2fe=testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe., dad01c63aab200f54826187889d7bb3a=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a., 0720db5474d7242cd64e0ebf2bd06ad8=hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8., 84e136ef3c363cd7fdbf7742a369e042=hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042., edb651d448a6733589b5e22423583e86=testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., 683f368212c8f5a82c099782900ab05b=testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b., 2c42380c364991ffc7b56aaffd602a09=testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09.} 2024-11-27T07:38:22,657 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 0720db5474d7242cd64e0ebf2bd06ad8, 1588230740, 2c42380c364991ffc7b56aaffd602a09, 65aaf181dbb3f882270fa6339a32aa32, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, c43ecc1a0eee0e96274e94265f82f2fe, dad01c63aab200f54826187889d7bb3a, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:22,674 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScannerAsync/65aaf181dbb3f882270fa6339a32aa32/.tmp/testFamily/7d2a92c8873a4d7f97e3ca732cea1fdb 2024-11-27T07:38:22,689 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScannerAsync/65aaf181dbb3f882270fa6339a32aa32/.tmp/testFamily/7d2a92c8873a4d7f97e3ca732cea1fdb as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScannerAsync/65aaf181dbb3f882270fa6339a32aa32/testFamily/7d2a92c8873a4d7f97e3ca732cea1fdb 2024-11-27T07:38:22,698 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScannerAsync/65aaf181dbb3f882270fa6339a32aa32/testFamily/7d2a92c8873a4d7f97e3ca732cea1fdb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T07:38:22,699 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 65aaf181dbb3f882270fa6339a32aa32 in 448ms, sequenceid=8, compaction requested=false 2024-11-27T07:38:22,704 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScannerAsync/65aaf181dbb3f882270fa6339a32aa32/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T07:38:22,705 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. 2024-11-27T07:38:22,705 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 65aaf181dbb3f882270fa6339a32aa32: Waiting for close lock at 1732693102251Running coprocessor pre-close hooks at 1732693102251Disabling compacts and flushes for region at 1732693102251Disabling writes for close at 1732693102251Obtaining lock to block concurrent updates at 1732693102251Preparing flush snapshotting stores in 65aaf181dbb3f882270fa6339a32aa32 at 1732693102251Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732693102251Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. at 1732693102252 (+1 ms)Flushing 65aaf181dbb3f882270fa6339a32aa32/testFamily: creating writer at 1732693102252Flushing 65aaf181dbb3f882270fa6339a32aa32/testFamily: appending metadata at 1732693102267 (+15 ms)Flushing 65aaf181dbb3f882270fa6339a32aa32/testFamily: closing flushed file at 1732693102267Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d2a9c21: reopening flushed file at 1732693102688 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 65aaf181dbb3f882270fa6339a32aa32 in 448ms, sequenceid=8, compaction requested=false at 1732693102699 (+11 ms)Writing region close event to WAL at 1732693102700 (+1 ms)Running coprocessor post-close hooks at 1732693102704 (+4 ms)Closed at 1732693102705 (+1 ms) 2024-11-27T07:38:22,705 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732693095495.65aaf181dbb3f882270fa6339a32aa32. 2024-11-27T07:38:22,705 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c43ecc1a0eee0e96274e94265f82f2fe, disabling compactions & flushes 2024-11-27T07:38:22,705 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. 2024-11-27T07:38:22,705 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. 2024-11-27T07:38:22,705 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. after waiting 0 ms 2024-11-27T07:38:22,705 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. 2024-11-27T07:38:22,705 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c43ecc1a0eee0e96274e94265f82f2fe 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T07:38:22,722 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNext/c43ecc1a0eee0e96274e94265f82f2fe/.tmp/testFamily/4297325dedec43028268f29c0526f0f4 is 61, key is row-0/testFamily:testQualifier/1732693074227/Put/seqid=0 2024-11-27T07:38:22,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741861_1037 (size=5049) 2024-11-27T07:38:22,858 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 0720db5474d7242cd64e0ebf2bd06ad8, 1588230740, 2c42380c364991ffc7b56aaffd602a09, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, c43ecc1a0eee0e96274e94265f82f2fe, dad01c63aab200f54826187889d7bb3a, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:23,003 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/.tmp/table/dc705a4ddf4744f4ae1c7ca21a5ac060 2024-11-27T07:38:23,017 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/.tmp/info/bcbeebe8a52f495cb41481a89f68ccc4 as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/info/bcbeebe8a52f495cb41481a89f68ccc4 2024-11-27T07:38:23,025 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/info/bcbeebe8a52f495cb41481a89f68ccc4, entries=100, sequenceid=56, filesize=20.1 K 2024-11-27T07:38:23,027 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/.tmp/ns/da22a1aaacca44b2b2d87e05200551de as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/ns/da22a1aaacca44b2b2d87e05200551de 2024-11-27T07:38:23,035 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/ns/da22a1aaacca44b2b2d87e05200551de, entries=2, sequenceid=56, filesize=5.0 K 2024-11-27T07:38:23,037 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/.tmp/table/dc705a4ddf4744f4ae1c7ca21a5ac060 as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/table/dc705a4ddf4744f4ae1c7ca21a5ac060 2024-11-27T07:38:23,045 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/table/dc705a4ddf4744f4ae1c7ca21a5ac060, entries=20, sequenceid=56, filesize=6.5 K 2024-11-27T07:38:23,046 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1391ms, sequenceid=56, compaction requested=false 2024-11-27T07:38:23,051 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-27T07:38:23,052 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T07:38:23,052 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T07:38:23,053 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732693101655Running coprocessor pre-close hooks at 1732693101655Disabling compacts and flushes for region at 1732693101655Disabling writes for close at 1732693101655Obtaining lock to block concurrent updates at 1732693101658 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1732693101658Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732693101666 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732693101667 (+1 ms)Flushing 1588230740/info: creating writer at 1732693101669 (+2 ms)Flushing 1588230740/info: appending metadata at 1732693101709 (+40 ms)Flushing 1588230740/info: closing flushed file at 1732693101709Flushing 1588230740/ns: creating writer at 1732693102144 (+435 ms)Flushing 1588230740/ns: appending metadata at 1732693102159 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732693102159Flushing 1588230740/table: creating writer at 1732693102581 (+422 ms)Flushing 1588230740/table: appending metadata at 1732693102595 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732693102595Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c0da783: reopening flushed file at 1732693103016 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bcc509: reopening flushed file at 1732693103026 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31c13740: reopening flushed file at 1732693103035 (+9 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1391ms, sequenceid=56, compaction requested=false at 1732693103046 (+11 ms)Writing region close event to WAL at 1732693103047 (+1 ms)Running coprocessor post-close hooks at 1732693103052 (+5 ms)Closed at 1732693103052 2024-11-27T07:38:23,053 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T07:38:23,058 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 0720db5474d7242cd64e0ebf2bd06ad8, 2c42380c364991ffc7b56aaffd602a09, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, c43ecc1a0eee0e96274e94265f82f2fe, dad01c63aab200f54826187889d7bb3a, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:23,128 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNext/c43ecc1a0eee0e96274e94265f82f2fe/.tmp/testFamily/4297325dedec43028268f29c0526f0f4 2024-11-27T07:38:23,139 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNext/c43ecc1a0eee0e96274e94265f82f2fe/.tmp/testFamily/4297325dedec43028268f29c0526f0f4 as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNext/c43ecc1a0eee0e96274e94265f82f2fe/testFamily/4297325dedec43028268f29c0526f0f4 2024-11-27T07:38:23,147 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNext/c43ecc1a0eee0e96274e94265f82f2fe/testFamily/4297325dedec43028268f29c0526f0f4, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T07:38:23,149 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c43ecc1a0eee0e96274e94265f82f2fe in 444ms, sequenceid=8, compaction requested=false 2024-11-27T07:38:23,154 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNext/c43ecc1a0eee0e96274e94265f82f2fe/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T07:38:23,155 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. 2024-11-27T07:38:23,156 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c43ecc1a0eee0e96274e94265f82f2fe: Waiting for close lock at 1732693102705Running coprocessor pre-close hooks at 1732693102705Disabling compacts and flushes for region at 1732693102705Disabling writes for close at 1732693102705Obtaining lock to block concurrent updates at 1732693102705Preparing flush snapshotting stores in c43ecc1a0eee0e96274e94265f82f2fe at 1732693102705Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732693102706 (+1 ms)Flushing stores of testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. at 1732693102706Flushing c43ecc1a0eee0e96274e94265f82f2fe/testFamily: creating writer at 1732693102706Flushing c43ecc1a0eee0e96274e94265f82f2fe/testFamily: appending metadata at 1732693102722 (+16 ms)Flushing c43ecc1a0eee0e96274e94265f82f2fe/testFamily: closing flushed file at 1732693102722Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73dff391: reopening flushed file at 1732693103138 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c43ecc1a0eee0e96274e94265f82f2fe in 444ms, sequenceid=8, compaction requested=false at 1732693103149 (+11 ms)Writing region close event to WAL at 1732693103150 (+1 ms)Running coprocessor post-close hooks at 1732693103155 (+5 ms)Closed at 1732693103155 2024-11-27T07:38:23,156 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732693071154.c43ecc1a0eee0e96274e94265f82f2fe. 2024-11-27T07:38:23,156 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dad01c63aab200f54826187889d7bb3a, disabling compactions & flushes 2024-11-27T07:38:23,156 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. 2024-11-27T07:38:23,156 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. 2024-11-27T07:38:23,156 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. after waiting 0 ms 2024-11-27T07:38:23,156 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. 2024-11-27T07:38:23,156 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing dad01c63aab200f54826187889d7bb3a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T07:38:23,178 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/dad01c63aab200f54826187889d7bb3a/.tmp/testFamily/e329284fda6e4916ac2686b9f832c7ec is 61, key is row-0/testFamily:testQualifier/1732693080338/Put/seqid=0 2024-11-27T07:38:23,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741862_1038 (size=5049) 2024-11-27T07:38:23,258 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 0720db5474d7242cd64e0ebf2bd06ad8, 2c42380c364991ffc7b56aaffd602a09, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, dad01c63aab200f54826187889d7bb3a, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:23,459 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 0720db5474d7242cd64e0ebf2bd06ad8, 2c42380c364991ffc7b56aaffd602a09, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, dad01c63aab200f54826187889d7bb3a, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:23,585 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/dad01c63aab200f54826187889d7bb3a/.tmp/testFamily/e329284fda6e4916ac2686b9f832c7ec 2024-11-27T07:38:23,598 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/dad01c63aab200f54826187889d7bb3a/.tmp/testFamily/e329284fda6e4916ac2686b9f832c7ec as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/dad01c63aab200f54826187889d7bb3a/testFamily/e329284fda6e4916ac2686b9f832c7ec 2024-11-27T07:38:23,607 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/dad01c63aab200f54826187889d7bb3a/testFamily/e329284fda6e4916ac2686b9f832c7ec, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T07:38:23,608 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for dad01c63aab200f54826187889d7bb3a in 452ms, sequenceid=8, compaction requested=false 2024-11-27T07:38:23,612 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/dad01c63aab200f54826187889d7bb3a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T07:38:23,613 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. 2024-11-27T07:38:23,613 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dad01c63aab200f54826187889d7bb3a: Waiting for close lock at 1732693103156Running coprocessor pre-close hooks at 1732693103156Disabling compacts and flushes for region at 1732693103156Disabling writes for close at 1732693103156Obtaining lock to block concurrent updates at 1732693103156Preparing flush snapshotting stores in dad01c63aab200f54826187889d7bb3a at 1732693103156Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732693103157 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. at 1732693103157Flushing dad01c63aab200f54826187889d7bb3a/testFamily: creating writer at 1732693103157Flushing dad01c63aab200f54826187889d7bb3a/testFamily: appending metadata at 1732693103177 (+20 ms)Flushing dad01c63aab200f54826187889d7bb3a/testFamily: closing flushed file at 1732693103177Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f17bb1e: reopening flushed file at 1732693103597 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for dad01c63aab200f54826187889d7bb3a in 452ms, sequenceid=8, compaction requested=false at 1732693103608 (+11 ms)Writing region close event to WAL at 1732693103608Running coprocessor post-close hooks at 1732693103613 (+5 ms)Closed at 1732693103613 2024-11-27T07:38:23,613 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732693077285.dad01c63aab200f54826187889d7bb3a. 2024-11-27T07:38:23,613 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0720db5474d7242cd64e0ebf2bd06ad8, disabling compactions & flushes 2024-11-27T07:38:23,613 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. 2024-11-27T07:38:23,613 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. 2024-11-27T07:38:23,613 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. after waiting 0 ms 2024-11-27T07:38:23,614 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. 2024-11-27T07:38:23,614 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0720db5474d7242cd64e0ebf2bd06ad8 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T07:38:23,632 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScanner/0720db5474d7242cd64e0ebf2bd06ad8/.tmp/testFamily/632c6d65ed4142e784a2e9e7feff82d3 is 61, key is row-0/testFamily:testQualifier/1732693042715/Put/seqid=0 2024-11-27T07:38:23,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741863_1039 (size=5049) 2024-11-27T07:38:23,659 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-11-27T07:38:23,659 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1325): Online Regions={0720db5474d7242cd64e0ebf2bd06ad8=hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8., 84e136ef3c363cd7fdbf7742a369e042=hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042., edb651d448a6733589b5e22423583e86=testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., 683f368212c8f5a82c099782900ab05b=testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b., 2c42380c364991ffc7b56aaffd602a09=testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09.} 2024-11-27T07:38:23,659 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 0720db5474d7242cd64e0ebf2bd06ad8, 2c42380c364991ffc7b56aaffd602a09, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:23,860 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 0720db5474d7242cd64e0ebf2bd06ad8, 2c42380c364991ffc7b56aaffd602a09, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:24,040 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScanner/0720db5474d7242cd64e0ebf2bd06ad8/.tmp/testFamily/632c6d65ed4142e784a2e9e7feff82d3 2024-11-27T07:38:24,056 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScanner/0720db5474d7242cd64e0ebf2bd06ad8/.tmp/testFamily/632c6d65ed4142e784a2e9e7feff82d3 as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScanner/0720db5474d7242cd64e0ebf2bd06ad8/testFamily/632c6d65ed4142e784a2e9e7feff82d3 2024-11-27T07:38:24,060 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 0720db5474d7242cd64e0ebf2bd06ad8, 2c42380c364991ffc7b56aaffd602a09, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:24,065 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScanner/0720db5474d7242cd64e0ebf2bd06ad8/testFamily/632c6d65ed4142e784a2e9e7feff82d3, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T07:38:24,066 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0720db5474d7242cd64e0ebf2bd06ad8 in 452ms, sequenceid=8, compaction requested=false 2024-11-27T07:38:24,070 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnOpenScanner/0720db5474d7242cd64e0ebf2bd06ad8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T07:38:24,071 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. 2024-11-27T07:38:24,071 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0720db5474d7242cd64e0ebf2bd06ad8: Waiting for close lock at 1732693103613Running coprocessor pre-close hooks at 1732693103613Disabling compacts and flushes for region at 1732693103613Disabling writes for close at 1732693103614 (+1 ms)Obtaining lock to block concurrent updates at 1732693103614Preparing flush snapshotting stores in 0720db5474d7242cd64e0ebf2bd06ad8 at 1732693103614Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732693103614Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. at 1732693103615 (+1 ms)Flushing 0720db5474d7242cd64e0ebf2bd06ad8/testFamily: creating writer at 1732693103615Flushing 0720db5474d7242cd64e0ebf2bd06ad8/testFamily: appending metadata at 1732693103632 (+17 ms)Flushing 0720db5474d7242cd64e0ebf2bd06ad8/testFamily: closing flushed file at 1732693103632Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a212ba8: reopening flushed file at 1732693104055 (+423 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0720db5474d7242cd64e0ebf2bd06ad8 in 452ms, sequenceid=8, compaction requested=false at 1732693104066 (+11 ms)Writing region close event to WAL at 1732693104066Running coprocessor post-close hooks at 1732693104071 (+5 ms)Closed at 1732693104071 2024-11-27T07:38:24,072 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732693039668.0720db5474d7242cd64e0ebf2bd06ad8. 2024-11-27T07:38:24,072 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 84e136ef3c363cd7fdbf7742a369e042, disabling compactions & flushes 2024-11-27T07:38:24,072 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. 2024-11-27T07:38:24,072 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. 2024-11-27T07:38:24,072 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. after waiting 0 ms 2024-11-27T07:38:24,072 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. 2024-11-27T07:38:24,072 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 84e136ef3c363cd7fdbf7742a369e042 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T07:38:24,087 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNextAsync/84e136ef3c363cd7fdbf7742a369e042/.tmp/testFamily/c5dc9666f10642f9bed2a2e69654d002 is 61, key is row-0/testFamily:testQualifier/1732693051812/Put/seqid=0 2024-11-27T07:38:24,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741864_1040 (size=5049) 2024-11-27T07:38:24,261 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 2c42380c364991ffc7b56aaffd602a09, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:24,461 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 2c42380c364991ffc7b56aaffd602a09, 683f368212c8f5a82c099782900ab05b, 84e136ef3c363cd7fdbf7742a369e042, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:24,492 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNextAsync/84e136ef3c363cd7fdbf7742a369e042/.tmp/testFamily/c5dc9666f10642f9bed2a2e69654d002 2024-11-27T07:38:24,504 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNextAsync/84e136ef3c363cd7fdbf7742a369e042/.tmp/testFamily/c5dc9666f10642f9bed2a2e69654d002 as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNextAsync/84e136ef3c363cd7fdbf7742a369e042/testFamily/c5dc9666f10642f9bed2a2e69654d002 2024-11-27T07:38:24,513 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNextAsync/84e136ef3c363cd7fdbf7742a369e042/testFamily/c5dc9666f10642f9bed2a2e69654d002, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T07:38:24,515 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 84e136ef3c363cd7fdbf7742a369e042 in 443ms, sequenceid=8, compaction requested=false 2024-11-27T07:38:24,521 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/hbase/testMetaScanTimeoutOnNextAsync/84e136ef3c363cd7fdbf7742a369e042/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T07:38:24,522 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. 2024-11-27T07:38:24,522 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 84e136ef3c363cd7fdbf7742a369e042: Waiting for close lock at 1732693104072Running coprocessor pre-close hooks at 1732693104072Disabling compacts and flushes for region at 1732693104072Disabling writes for close at 1732693104072Obtaining lock to block concurrent updates at 1732693104072Preparing flush snapshotting stores in 84e136ef3c363cd7fdbf7742a369e042 at 1732693104072Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732693104072Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. at 1732693104073 (+1 ms)Flushing 84e136ef3c363cd7fdbf7742a369e042/testFamily: creating writer at 1732693104073Flushing 84e136ef3c363cd7fdbf7742a369e042/testFamily: appending metadata at 1732693104087 (+14 ms)Flushing 84e136ef3c363cd7fdbf7742a369e042/testFamily: closing flushed file at 1732693104087Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6155d2b5: reopening flushed file at 1732693104502 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 84e136ef3c363cd7fdbf7742a369e042 in 443ms, sequenceid=8, compaction requested=false at 1732693104515 (+13 ms)Writing region close event to WAL at 1732693104516 (+1 ms)Running coprocessor post-close hooks at 1732693104522 (+6 ms)Closed at 1732693104522 2024-11-27T07:38:24,522 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732693048768.84e136ef3c363cd7fdbf7742a369e042. 2024-11-27T07:38:24,523 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing edb651d448a6733589b5e22423583e86, disabling compactions & flushes 2024-11-27T07:38:24,523 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. 2024-11-27T07:38:24,523 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. 2024-11-27T07:38:24,523 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. after waiting 0 ms 2024-11-27T07:38:24,523 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. 2024-11-27T07:38:24,523 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing edb651d448a6733589b5e22423583e86 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T07:38:24,538 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScanner/edb651d448a6733589b5e22423583e86/.tmp/testFamily/0a09a7c2e3c845539c9c4eec6d3cd347 is 61, key is row-0/testFamily:testQualifier/1732693061999/Put/seqid=0 2024-11-27T07:38:24,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741865_1041 (size=5049) 2024-11-27T07:38:24,662 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-27T07:38:24,662 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1325): Online Regions={edb651d448a6733589b5e22423583e86=testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., 683f368212c8f5a82c099782900ab05b=testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b., 2c42380c364991ffc7b56aaffd602a09=testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09.} 2024-11-27T07:38:24,662 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 2c42380c364991ffc7b56aaffd602a09, 683f368212c8f5a82c099782900ab05b, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:24,863 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 2c42380c364991ffc7b56aaffd602a09, 683f368212c8f5a82c099782900ab05b, edb651d448a6733589b5e22423583e86 2024-11-27T07:38:24,944 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScanner/edb651d448a6733589b5e22423583e86/.tmp/testFamily/0a09a7c2e3c845539c9c4eec6d3cd347 2024-11-27T07:38:24,957 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScanner/edb651d448a6733589b5e22423583e86/.tmp/testFamily/0a09a7c2e3c845539c9c4eec6d3cd347 as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScanner/edb651d448a6733589b5e22423583e86/testFamily/0a09a7c2e3c845539c9c4eec6d3cd347 2024-11-27T07:38:24,965 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScanner/edb651d448a6733589b5e22423583e86/testFamily/0a09a7c2e3c845539c9c4eec6d3cd347, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T07:38:24,966 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for edb651d448a6733589b5e22423583e86 in 443ms, sequenceid=8, compaction requested=false 2024-11-27T07:38:24,971 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnOpenScanner/edb651d448a6733589b5e22423583e86/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T07:38:24,972 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. 2024-11-27T07:38:24,972 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for edb651d448a6733589b5e22423583e86: Waiting for close lock at 1732693104523Running coprocessor pre-close hooks at 1732693104523Disabling compacts and flushes for region at 1732693104523Disabling writes for close at 1732693104523Obtaining lock to block concurrent updates at 1732693104523Preparing flush snapshotting stores in edb651d448a6733589b5e22423583e86 at 1732693104523Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732693104523Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. at 1732693104524 (+1 ms)Flushing edb651d448a6733589b5e22423583e86/testFamily: creating writer at 1732693104524Flushing edb651d448a6733589b5e22423583e86/testFamily: appending metadata at 1732693104538 (+14 ms)Flushing edb651d448a6733589b5e22423583e86/testFamily: closing flushed file at 1732693104538Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f11ebb8: reopening flushed file at 1732693104955 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for edb651d448a6733589b5e22423583e86 in 443ms, sequenceid=8, compaction requested=false at 1732693104966 (+11 ms)Writing region close event to WAL at 1732693104967 (+1 ms)Running coprocessor post-close hooks at 1732693104972 (+5 ms)Closed at 1732693104972 2024-11-27T07:38:24,972 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732693058942.edb651d448a6733589b5e22423583e86. 2024-11-27T07:38:24,972 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 683f368212c8f5a82c099782900ab05b, disabling compactions & flushes 2024-11-27T07:38:24,972 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. 2024-11-27T07:38:24,972 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. 2024-11-27T07:38:24,972 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. after waiting 0 ms 2024-11-27T07:38:24,973 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. 2024-11-27T07:38:24,973 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 683f368212c8f5a82c099782900ab05b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T07:38:24,990 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNextAsync/683f368212c8f5a82c099782900ab05b/.tmp/testFamily/116f3605b5e34a1f9b651722628e2bd1 is 61, key is row-0/testFamily:testQualifier/1732693068103/Put/seqid=0 2024-11-27T07:38:24,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741866_1042 (size=5049) 2024-11-27T07:38:25,063 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 2c42380c364991ffc7b56aaffd602a09, 683f368212c8f5a82c099782900ab05b 2024-11-27T07:38:25,264 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 2c42380c364991ffc7b56aaffd602a09, 683f368212c8f5a82c099782900ab05b 2024-11-27T07:38:25,397 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNextAsync/683f368212c8f5a82c099782900ab05b/.tmp/testFamily/116f3605b5e34a1f9b651722628e2bd1 2024-11-27T07:38:25,411 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNextAsync/683f368212c8f5a82c099782900ab05b/.tmp/testFamily/116f3605b5e34a1f9b651722628e2bd1 as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNextAsync/683f368212c8f5a82c099782900ab05b/testFamily/116f3605b5e34a1f9b651722628e2bd1 2024-11-27T07:38:25,419 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNextAsync/683f368212c8f5a82c099782900ab05b/testFamily/116f3605b5e34a1f9b651722628e2bd1, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T07:38:25,420 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 683f368212c8f5a82c099782900ab05b in 447ms, sequenceid=8, compaction requested=false 2024-11-27T07:38:25,425 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testNormalScanTimeoutOnNextAsync/683f368212c8f5a82c099782900ab05b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T07:38:25,426 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. 2024-11-27T07:38:25,426 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 683f368212c8f5a82c099782900ab05b: Waiting for close lock at 1732693104972Running coprocessor pre-close hooks at 1732693104972Disabling compacts and flushes for region at 1732693104972Disabling writes for close at 1732693104973 (+1 ms)Obtaining lock to block concurrent updates at 1732693104973Preparing flush snapshotting stores in 683f368212c8f5a82c099782900ab05b at 1732693104973Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732693104973Flushing stores of testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. at 1732693104974 (+1 ms)Flushing 683f368212c8f5a82c099782900ab05b/testFamily: creating writer at 1732693104974Flushing 683f368212c8f5a82c099782900ab05b/testFamily: appending metadata at 1732693104990 (+16 ms)Flushing 683f368212c8f5a82c099782900ab05b/testFamily: closing flushed file at 1732693104990Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@681ade11: reopening flushed file at 1732693105410 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 683f368212c8f5a82c099782900ab05b in 447ms, sequenceid=8, compaction requested=false at 1732693105420 (+10 ms)Writing region close event to WAL at 1732693105421 (+1 ms)Running coprocessor post-close hooks at 1732693105426 (+5 ms)Closed at 1732693105426 2024-11-27T07:38:25,426 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732693065052.683f368212c8f5a82c099782900ab05b. 2024-11-27T07:38:25,426 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2c42380c364991ffc7b56aaffd602a09, disabling compactions & flushes 2024-11-27T07:38:25,426 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. 2024-11-27T07:38:25,426 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. 2024-11-27T07:38:25,426 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. after waiting 0 ms 2024-11-27T07:38:25,426 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. 2024-11-27T07:38:25,426 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2c42380c364991ffc7b56aaffd602a09 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T07:38:25,442 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/2c42380c364991ffc7b56aaffd602a09/.tmp/testFamily/5bef9901518645aa86170f2145cd7ffa is 61, key is row-0/testFamily:testQualifier/1732693039578/Put/seqid=0 2024-11-27T07:38:25,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741867_1043 (size=5049) 2024-11-27T07:38:25,464 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:38:25,664 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T07:38:25,665 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1325): Online Regions={2c42380c364991ffc7b56aaffd602a09=testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09.} 2024-11-27T07:38:25,665 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:38:25,850 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/2c42380c364991ffc7b56aaffd602a09/.tmp/testFamily/5bef9901518645aa86170f2145cd7ffa 2024-11-27T07:38:25,864 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/2c42380c364991ffc7b56aaffd602a09/.tmp/testFamily/5bef9901518645aa86170f2145cd7ffa as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/2c42380c364991ffc7b56aaffd602a09/testFamily/5bef9901518645aa86170f2145cd7ffa 2024-11-27T07:38:25,865 DEBUG [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1351): Waiting on 2c42380c364991ffc7b56aaffd602a09 2024-11-27T07:38:25,871 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/2c42380c364991ffc7b56aaffd602a09/testFamily/5bef9901518645aa86170f2145cd7ffa, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T07:38:25,872 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2c42380c364991ffc7b56aaffd602a09 in 446ms, sequenceid=8, compaction requested=false 2024-11-27T07:38:25,876 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/2c42380c364991ffc7b56aaffd602a09/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T07:38:25,877 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. 2024-11-27T07:38:25,877 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2c42380c364991ffc7b56aaffd602a09: Waiting for close lock at 1732693105426Running coprocessor pre-close hooks at 1732693105426Disabling compacts and flushes for region at 1732693105426Disabling writes for close at 1732693105426Obtaining lock to block concurrent updates at 1732693105426Preparing flush snapshotting stores in 2c42380c364991ffc7b56aaffd602a09 at 1732693105426Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732693105427 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. at 1732693105427Flushing 2c42380c364991ffc7b56aaffd602a09/testFamily: creating writer at 1732693105427Flushing 2c42380c364991ffc7b56aaffd602a09/testFamily: appending metadata at 1732693105442 (+15 ms)Flushing 2c42380c364991ffc7b56aaffd602a09/testFamily: closing flushed file at 1732693105442Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f97f21a: reopening flushed file at 1732693105863 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2c42380c364991ffc7b56aaffd602a09 in 446ms, sequenceid=8, compaction requested=false at 1732693105872 (+9 ms)Writing region close event to WAL at 1732693105872Running coprocessor post-close hooks at 1732693105877 (+5 ms)Closed at 1732693105877 2024-11-27T07:38:25,877 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732693036507.2c42380c364991ffc7b56aaffd602a09. 2024-11-27T07:38:26,066 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(976): stopping server 289954c9202d,41527,1732693002367; all regions closed. 2024-11-27T07:38:26,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741834_1010 (size=19479) 2024-11-27T07:38:26,115 DEBUG [RS:0;289954c9202d:41527 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/oldWALs 2024-11-27T07:38:26,115 INFO [RS:0;289954c9202d:41527 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 289954c9202d%2C41527%2C1732693002367.meta:.meta(num 1732693036037) 2024-11-27T07:38:26,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741833_1009 (size=20031) 2024-11-27T07:38:26,118 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/WALs/289954c9202d,41527,1732693002367/289954c9202d%2C41527%2C1732693002367.1732693035632 not finished, retry = 0 2024-11-27T07:38:26,226 DEBUG [RS:0;289954c9202d:41527 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/oldWALs 2024-11-27T07:38:26,226 INFO [RS:0;289954c9202d:41527 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 289954c9202d%2C41527%2C1732693002367:(num 1732693035632) 2024-11-27T07:38:26,227 DEBUG [RS:0;289954c9202d:41527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:38:26,227 INFO [RS:0;289954c9202d:41527 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T07:38:26,227 INFO [RS:0;289954c9202d:41527 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T07:38:26,228 INFO [RS:0;289954c9202d:41527 {}] hbase.ChoreService(370): Chore service for: regionserver/289954c9202d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-27T07:38:26,228 INFO [RS:0;289954c9202d:41527 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T07:38:26,228 INFO [regionserver/289954c9202d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T07:38:26,231 INFO [RS:0;289954c9202d:41527 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41527 2024-11-27T07:38:26,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/289954c9202d,41527,1732693002367 2024-11-27T07:38:26,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T07:38:26,256 INFO [RS:0;289954c9202d:41527 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T07:38:26,267 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [289954c9202d,41527,1732693002367] 2024-11-27T07:38:26,277 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/289954c9202d,41527,1732693002367 already deleted, retry=false 2024-11-27T07:38:26,277 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 289954c9202d,41527,1732693002367 expired; onlineServers=0 2024-11-27T07:38:26,277 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '289954c9202d,41501,1732693001213' ***** 2024-11-27T07:38:26,278 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T07:38:26,278 INFO [M:0;289954c9202d:41501 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T07:38:26,278 INFO [M:0;289954c9202d:41501 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T07:38:26,278 DEBUG [M:0;289954c9202d:41501 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T07:38:26,279 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T07:38:26,279 DEBUG [M:0;289954c9202d:41501 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T07:38:26,279 DEBUG [master/289954c9202d:0:becomeActiveMaster-HFileCleaner.small.0-1732693005426 {}] cleaner.HFileCleaner(306): Exit Thread[master/289954c9202d:0:becomeActiveMaster-HFileCleaner.small.0-1732693005426,5,FailOnTimeoutGroup] 2024-11-27T07:38:26,279 DEBUG [master/289954c9202d:0:becomeActiveMaster-HFileCleaner.large.0-1732693005422 {}] cleaner.HFileCleaner(306): Exit Thread[master/289954c9202d:0:becomeActiveMaster-HFileCleaner.large.0-1732693005422,5,FailOnTimeoutGroup] 2024-11-27T07:38:26,279 INFO [M:0;289954c9202d:41501 {}] hbase.ChoreService(370): Chore service for: master/289954c9202d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T07:38:26,279 INFO [M:0;289954c9202d:41501 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T07:38:26,280 DEBUG [M:0;289954c9202d:41501 {}] master.HMaster(1795): Stopping service threads 2024-11-27T07:38:26,280 INFO [M:0;289954c9202d:41501 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T07:38:26,280 INFO [M:0;289954c9202d:41501 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T07:38:26,281 INFO [M:0;289954c9202d:41501 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T07:38:26,282 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T07:38:26,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T07:38:26,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:38:26,288 DEBUG [M:0;289954c9202d:41501 {}] zookeeper.ZKUtil(347): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T07:38:26,288 WARN [M:0;289954c9202d:41501 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T07:38:26,289 INFO [M:0;289954c9202d:41501 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/.lastflushedseqids 2024-11-27T07:38:26,293 INFO [regionserver/289954c9202d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T07:38:26,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741868_1044 (size=649) 2024-11-27T07:38:26,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T07:38:26,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41527-0x1017b38fc630001, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T07:38:26,368 INFO [RS:0;289954c9202d:41527 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T07:38:26,369 INFO [RS:0;289954c9202d:41527 {}] regionserver.HRegionServer(1031): Exiting; stopping=289954c9202d,41527,1732693002367; zookeeper connection closed. 2024-11-27T07:38:26,370 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@25271586 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@25271586 2024-11-27T07:38:26,371 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-27T07:38:26,708 INFO [M:0;289954c9202d:41501 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T07:38:26,709 INFO [M:0;289954c9202d:41501 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T07:38:26,709 DEBUG [M:0;289954c9202d:41501 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T07:38:26,709 INFO [M:0;289954c9202d:41501 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:38:26,709 DEBUG [M:0;289954c9202d:41501 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:38:26,709 DEBUG [M:0;289954c9202d:41501 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T07:38:26,709 DEBUG [M:0;289954c9202d:41501 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:38:26,710 INFO [M:0;289954c9202d:41501 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.11 KB heapSize=191.02 KB 2024-11-27T07:38:26,734 DEBUG [M:0;289954c9202d:41501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d286ea6631cb4306baeffb9d13c799b7 is 82, key is hbase:meta,,1/info:regioninfo/1732693036115/Put/seqid=0 2024-11-27T07:38:26,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741869_1045 (size=5672) 2024-11-27T07:38:27,141 INFO [M:0;289954c9202d:41501 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d286ea6631cb4306baeffb9d13c799b7 2024-11-27T07:38:27,176 DEBUG [M:0;289954c9202d:41501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9c768b9fb25549e5827dd22522eb8b98 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732693037769/Put/seqid=0 2024-11-27T07:38:27,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741870_1046 (size=14079) 2024-11-27T07:38:27,298 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:38:27,584 INFO [M:0;289954c9202d:41501 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.55 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9c768b9fb25549e5827dd22522eb8b98 2024-11-27T07:38:27,597 INFO [M:0;289954c9202d:41501 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9c768b9fb25549e5827dd22522eb8b98 2024-11-27T07:38:27,614 DEBUG [M:0;289954c9202d:41501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/840d0ed359d94868b0a0635152985a49 is 69, key is 289954c9202d,41527,1732693002367/rs:state/1732693005473/Put/seqid=0 2024-11-27T07:38:27,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741871_1047 (size=5156) 2024-11-27T07:38:28,022 INFO [M:0;289954c9202d:41501 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/840d0ed359d94868b0a0635152985a49 2024-11-27T07:38:28,038 DEBUG [M:0;289954c9202d:41501 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d286ea6631cb4306baeffb9d13c799b7 as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d286ea6631cb4306baeffb9d13c799b7 2024-11-27T07:38:28,045 INFO [M:0;289954c9202d:41501 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d286ea6631cb4306baeffb9d13c799b7, entries=8, sequenceid=317, filesize=5.5 K 2024-11-27T07:38:28,047 DEBUG [M:0;289954c9202d:41501 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9c768b9fb25549e5827dd22522eb8b98 as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9c768b9fb25549e5827dd22522eb8b98 2024-11-27T07:38:28,054 INFO [M:0;289954c9202d:41501 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9c768b9fb25549e5827dd22522eb8b98 2024-11-27T07:38:28,055 INFO [M:0;289954c9202d:41501 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9c768b9fb25549e5827dd22522eb8b98, entries=33, sequenceid=317, filesize=13.7 K 2024-11-27T07:38:28,056 DEBUG [M:0;289954c9202d:41501 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/840d0ed359d94868b0a0635152985a49 as hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/840d0ed359d94868b0a0635152985a49 2024-11-27T07:38:28,064 INFO [M:0;289954c9202d:41501 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46073/user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/840d0ed359d94868b0a0635152985a49, entries=1, sequenceid=317, filesize=5.0 K 2024-11-27T07:38:28,065 INFO [M:0;289954c9202d:41501 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.11 KB/167023, heapSize ~190.72 KB/195296, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1356ms, sequenceid=317, compaction requested=false 2024-11-27T07:38:28,067 INFO [M:0;289954c9202d:41501 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:38:28,067 DEBUG [M:0;289954c9202d:41501 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732693106709Disabling compacts and flushes for region at 1732693106709Disabling writes for close at 1732693106709Obtaining lock to block concurrent updates at 1732693106710 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732693106710Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=167023, getHeapSize=195536, getOffHeapSize=0, getCellsCount=364 at 1732693106711 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732693106712 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732693106712Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732693106733 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732693106734 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732693107157 (+423 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732693107176 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732693107176Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732693107597 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732693107614 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732693107614Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d7f0606: reopening flushed file at 1732693108037 (+423 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34a05246: reopening flushed file at 1732693108046 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@474f5962: reopening flushed file at 1732693108055 (+9 ms)Finished flush of dataSize ~163.11 KB/167023, heapSize ~190.72 KB/195296, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1356ms, sequenceid=317, compaction requested=false at 1732693108065 (+10 ms)Writing region close event to WAL at 1732693108067 (+2 ms)Closed at 1732693108067 2024-11-27T07:38:28,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741830_1006 (size=188235) 2024-11-27T07:38:28,070 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/bd66c9a6-eae9-eb2c-c6c9-42414248b03c/MasterData/WALs/289954c9202d,41501,1732693001213/289954c9202d%2C41501%2C1732693001213.1732693004713 not finished, retry = 0 2024-11-27T07:38:28,172 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T07:38:28,172 INFO [M:0;289954c9202d:41501 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T07:38:28,172 INFO [M:0;289954c9202d:41501 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41501 2024-11-27T07:38:28,172 INFO [M:0;289954c9202d:41501 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T07:38:28,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T07:38:28,355 INFO [M:0;289954c9202d:41501 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T07:38:28,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41501-0x1017b38fc630000, quorum=127.0.0.1:56176, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T07:38:28,364 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6261b108{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T07:38:28,367 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@379257e4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T07:38:28,367 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T07:38:28,367 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48199fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T07:38:28,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29ee0fe9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/hadoop.log.dir/,STOPPED} 2024-11-27T07:38:28,370 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T07:38:28,370 WARN [BP-1981243590-172.17.0.3-1732692997009 heartbeating to localhost/127.0.0.1:46073 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T07:38:28,370 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T07:38:28,370 WARN [BP-1981243590-172.17.0.3-1732692997009 heartbeating to localhost/127.0.0.1:46073 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1981243590-172.17.0.3-1732692997009 (Datanode Uuid ec107a75-18f9-4cbf-9000-da7b0d22afbb) service to localhost/127.0.0.1:46073 2024-11-27T07:38:28,372 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/cluster_2a34800c-648a-d9f7-feb3-2a3237ffb0ee/data/data1/current/BP-1981243590-172.17.0.3-1732692997009 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T07:38:28,372 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/cluster_2a34800c-648a-d9f7-feb3-2a3237ffb0ee/data/data2/current/BP-1981243590-172.17.0.3-1732692997009 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T07:38:28,373 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T07:38:28,382 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4df87c21{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T07:38:28,383 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@29047fa7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T07:38:28,383 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T07:38:28,383 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5c1e5f71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T07:38:28,383 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3653a146{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/52c0b88c-3e29-01ce-6114-985fc47c80fd/hadoop.log.dir/,STOPPED} 2024-11-27T07:38:28,391 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T07:38:28,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down