2024-11-12 07:51:29,918 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 2024-11-12 07:51:29,937 main DEBUG Took 0.015646 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 07:51:29,937 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 07:51:29,938 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 07:51:29,940 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 07:51:29,942 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:51:29,953 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 07:51:29,991 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:51:29,993 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:51:29,994 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:51:29,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:51:29,995 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:51:29,995 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:51:29,997 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:51:29,997 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:51:29,998 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:51:29,998 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:51:30,000 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:51:30,000 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:51:30,001 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:51:30,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:51:30,002 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:51:30,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:51:30,003 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:51:30,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:51:30,004 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:51:30,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:51:30,005 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:51:30,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:51:30,006 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:51:30,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:51:30,007 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:51:30,007 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 07:51:30,009 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:51:30,011 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 07:51:30,014 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 07:51:30,015 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 07:51:30,017 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 07:51:30,017 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 07:51:30,028 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 07:51:30,031 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 07:51:30,034 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 07:51:30,034 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 07:51:30,035 main DEBUG createAppenders(={Console}) 2024-11-12 07:51:30,036 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 initialized 2024-11-12 07:51:30,036 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 2024-11-12 07:51:30,036 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 OK. 2024-11-12 07:51:30,037 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 07:51:30,037 main DEBUG OutputStream closed 2024-11-12 07:51:30,038 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 07:51:30,038 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 07:51:30,038 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@57cf54e1 OK 2024-11-12 07:51:30,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 07:51:30,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 07:51:30,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 07:51:30,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 07:51:30,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 07:51:30,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 07:51:30,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 07:51:30,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 07:51:30,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 07:51:30,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 07:51:30,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 07:51:30,150 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 07:51:30,150 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 07:51:30,151 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 07:51:30,151 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 07:51:30,152 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 07:51:30,152 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 07:51:30,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 07:51:30,156 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 07:51:30,157 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@1608bcbd) with optional ClassLoader: null 2024-11-12 07:51:30,158 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 07:51:30,159 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@1608bcbd] started OK. 2024-11-12T07:51:30,176 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-12 07:51:30,179 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 07:51:30,179 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T07:51:30,427 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c 2024-11-12T07:51:30,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T07:51:30,466 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/cluster_af29951c-f6f7-2c1e-a0a4-3fa84ccae668, deleteOnExit=true 2024-11-12T07:51:30,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T07:51:30,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/test.cache.data in system properties and HBase conf 2024-11-12T07:51:30,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T07:51:30,470 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/hadoop.log.dir in system properties and HBase conf 2024-11-12T07:51:30,471 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T07:51:30,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T07:51:30,472 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T07:51:30,593 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T07:51:30,718 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:51:30,724 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T07:51:30,725 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T07:51:30,725 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T07:51:30,726 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T07:51:30,727 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T07:51:30,727 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T07:51:30,728 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T07:51:30,729 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T07:51:30,729 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T07:51:30,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/nfs.dump.dir in system properties and HBase conf 2024-11-12T07:51:30,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/java.io.tmpdir in system properties and HBase conf 2024-11-12T07:51:30,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T07:51:30,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T07:51:30,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T07:51:31,824 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T07:51:31,912 INFO [Time-limited test {}] log.Log(170): Logging initialized @6122ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T07:51:31,999 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T07:51:32,106 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T07:51:32,151 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T07:51:32,151 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T07:51:32,153 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T07:51:32,174 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T07:51:32,179 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@24268d3a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/hadoop.log.dir/,AVAILABLE} 2024-11-12T07:51:32,180 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@549fd2eb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T07:51:32,463 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@b5692ae{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/java.io.tmpdir/jetty-localhost-34803-hadoop-hdfs-3_4_1-tests_jar-_-any-14292346550879571182/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T07:51:32,473 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1809bc91{HTTP/1.1, (http/1.1)}{localhost:34803} 2024-11-12T07:51:32,474 INFO [Time-limited test {}] server.Server(415): Started @6685ms 2024-11-12T07:51:33,039 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T07:51:33,047 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T07:51:33,048 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T07:51:33,048 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T07:51:33,048 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T07:51:33,049 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b440b02{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/hadoop.log.dir/,AVAILABLE} 2024-11-12T07:51:33,050 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d71513b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T07:51:33,171 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5b0930ac{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/java.io.tmpdir/jetty-localhost-40123-hadoop-hdfs-3_4_1-tests_jar-_-any-703179487645681591/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T07:51:33,172 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@257dd063{HTTP/1.1, (http/1.1)}{localhost:40123} 2024-11-12T07:51:33,172 INFO [Time-limited test {}] server.Server(415): Started @7383ms 2024-11-12T07:51:33,225 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T07:51:34,136 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/cluster_af29951c-f6f7-2c1e-a0a4-3fa84ccae668/data/data2/current/BP-945294674-172.17.0.2-1731397891410/current, will proceed with Du for space computation calculation, 2024-11-12T07:51:34,136 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/cluster_af29951c-f6f7-2c1e-a0a4-3fa84ccae668/data/data1/current/BP-945294674-172.17.0.2-1731397891410/current, will proceed with Du for space computation calculation, 2024-11-12T07:51:34,180 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T07:51:34,245 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2c1478f3e966ecb9 with lease ID 0x9de94ec97c01670f: Processing first storage report for DS-783c5011-81e9-4d06-b80e-db224bd39042 from datanode DatanodeRegistration(127.0.0.1:39471, datanodeUuid=ca631ca4-2819-4a29-b7dd-8ab31d1fd8e0, infoPort=44047, infoSecurePort=0, ipcPort=41763, storageInfo=lv=-57;cid=testClusterID;nsid=1509713719;c=1731397891410) 2024-11-12T07:51:34,247 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2c1478f3e966ecb9 with lease ID 0x9de94ec97c01670f: from storage DS-783c5011-81e9-4d06-b80e-db224bd39042 node DatanodeRegistration(127.0.0.1:39471, datanodeUuid=ca631ca4-2819-4a29-b7dd-8ab31d1fd8e0, infoPort=44047, infoSecurePort=0, ipcPort=41763, storageInfo=lv=-57;cid=testClusterID;nsid=1509713719;c=1731397891410), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-12T07:51:34,248 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2c1478f3e966ecb9 with lease ID 0x9de94ec97c01670f: Processing first storage report for DS-391d05a2-d5b1-4db7-af20-de16f9671f14 from datanode DatanodeRegistration(127.0.0.1:39471, datanodeUuid=ca631ca4-2819-4a29-b7dd-8ab31d1fd8e0, infoPort=44047, infoSecurePort=0, ipcPort=41763, storageInfo=lv=-57;cid=testClusterID;nsid=1509713719;c=1731397891410) 2024-11-12T07:51:34,248 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2c1478f3e966ecb9 with lease ID 0x9de94ec97c01670f: from storage DS-391d05a2-d5b1-4db7-af20-de16f9671f14 node DatanodeRegistration(127.0.0.1:39471, datanodeUuid=ca631ca4-2819-4a29-b7dd-8ab31d1fd8e0, infoPort=44047, infoSecurePort=0, ipcPort=41763, storageInfo=lv=-57;cid=testClusterID;nsid=1509713719;c=1731397891410), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T07:51:34,288 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c 2024-11-12T07:51:34,408 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/cluster_af29951c-f6f7-2c1e-a0a4-3fa84ccae668/zookeeper_0, clientPort=49256, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/cluster_af29951c-f6f7-2c1e-a0a4-3fa84ccae668/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/cluster_af29951c-f6f7-2c1e-a0a4-3fa84ccae668/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T07:51:34,421 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49256 2024-11-12T07:51:34,438 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T07:51:34,442 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T07:51:34,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741825_1001 (size=7) 2024-11-12T07:51:35,146 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a with version=8 2024-11-12T07:51:35,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/hbase-staging 2024-11-12T07:51:35,224 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T07:51:35,426 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f9fc96254ef8:0 server-side Connection retries=9 2024-11-12T07:51:35,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:51:35,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T07:51:35,440 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T07:51:35,440 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:51:35,440 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T07:51:35,564 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T07:51:35,615 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T07:51:35,623 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T07:51:35,626 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T07:51:35,647 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 38539 (auto-detected) 2024-11-12T07:51:35,648 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-12T07:51:35,664 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42363 2024-11-12T07:51:35,684 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42363 connecting to ZooKeeper ensemble=127.0.0.1:49256 2024-11-12T07:51:35,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:423630x0, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T07:51:35,793 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42363-0x1012f50d3b20000 connected 2024-11-12T07:51:35,923 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T07:51:35,926 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T07:51:35,934 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:51:35,938 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a, hbase.cluster.distributed=false 2024-11-12T07:51:35,961 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T07:51:35,966 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42363 2024-11-12T07:51:35,966 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42363 2024-11-12T07:51:35,966 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42363 2024-11-12T07:51:35,967 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42363 2024-11-12T07:51:35,967 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42363 2024-11-12T07:51:36,077 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f9fc96254ef8:0 server-side Connection retries=9 2024-11-12T07:51:36,079 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:51:36,079 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T07:51:36,079 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T07:51:36,080 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:51:36,080 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T07:51:36,083 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T07:51:36,086 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T07:51:36,087 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43507 2024-11-12T07:51:36,090 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43507 connecting to ZooKeeper ensemble=127.0.0.1:49256 2024-11-12T07:51:36,091 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T07:51:36,095 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T07:51:36,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:435070x0, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T07:51:36,143 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:435070x0, quorum=127.0.0.1:49256, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:51:36,143 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43507-0x1012f50d3b20001 connected 2024-11-12T07:51:36,148 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T07:51:36,158 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T07:51:36,161 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T07:51:36,167 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T07:51:36,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43507 2024-11-12T07:51:36,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43507 2024-11-12T07:51:36,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43507 2024-11-12T07:51:36,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43507 2024-11-12T07:51:36,170 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43507 2024-11-12T07:51:36,184 DEBUG [M:0;f9fc96254ef8:42363 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f9fc96254ef8:42363 2024-11-12T07:51:36,185 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f9fc96254ef8,42363,1731397895269 2024-11-12T07:51:36,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:51:36,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:51:36,228 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f9fc96254ef8,42363,1731397895269 2024-11-12T07:51:36,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T07:51:36,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:51:36,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:51:36,281 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T07:51:36,283 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f9fc96254ef8,42363,1731397895269 from backup master directory 2024-11-12T07:51:36,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:51:36,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f9fc96254ef8,42363,1731397895269 2024-11-12T07:51:36,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:51:36,314 WARN [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T07:51:36,314 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f9fc96254ef8,42363,1731397895269 2024-11-12T07:51:36,316 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T07:51:36,317 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T07:51:36,368 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/hbase.id] with ID: 5945df39-794d-4b95-a443-ecbda9762540 2024-11-12T07:51:36,368 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/.tmp/hbase.id 2024-11-12T07:51:36,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741826_1002 (size=42) 2024-11-12T07:51:36,780 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/.tmp/hbase.id]:[hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/hbase.id] 2024-11-12T07:51:36,827 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T07:51:36,832 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T07:51:36,852 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-12T07:51:36,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:51:36,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:51:36,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741827_1003 (size=196) 2024-11-12T07:51:37,363 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:51:37,370 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T07:51:37,395 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:51:37,402 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:51:37,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741828_1004 (size=1189) 2024-11-12T07:51:37,500 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store 2024-11-12T07:51:37,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741829_1005 (size=34) 2024-11-12T07:51:37,560 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T07:51:37,564 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:51:37,566 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T07:51:37,567 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:51:37,567 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:51:37,569 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T07:51:37,570 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:51:37,570 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:51:37,572 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731397897566Disabling compacts and flushes for region at 1731397897566Disabling writes for close at 1731397897570 (+4 ms)Writing region close event to WAL at 1731397897570Closed at 1731397897570 2024-11-12T07:51:37,575 WARN [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/.initializing 2024-11-12T07:51:37,576 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/WALs/f9fc96254ef8,42363,1731397895269 2024-11-12T07:51:37,590 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T07:51:37,614 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f9fc96254ef8%2C42363%2C1731397895269, suffix=, logDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/WALs/f9fc96254ef8,42363,1731397895269, archiveDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/oldWALs, maxLogs=10 2024-11-12T07:51:37,650 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/WALs/f9fc96254ef8,42363,1731397895269/f9fc96254ef8%2C42363%2C1731397895269.1731397897621, exclude list is [], retry=0 2024-11-12T07:51:37,679 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39471,DS-783c5011-81e9-4d06-b80e-db224bd39042,DISK] 2024-11-12T07:51:37,683 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T07:51:37,735 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/WALs/f9fc96254ef8,42363,1731397895269/f9fc96254ef8%2C42363%2C1731397895269.1731397897621 2024-11-12T07:51:37,740 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44047:44047)] 2024-11-12T07:51:37,740 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:51:37,741 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:51:37,747 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:51:37,749 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:51:37,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:51:37,839 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T07:51:37,844 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:51:37,848 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:51:37,849 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:51:37,853 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T07:51:37,854 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:51:37,855 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:51:37,856 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:51:37,859 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T07:51:37,860 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:51:37,861 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:51:37,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:51:37,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T07:51:37,866 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:51:37,868 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:51:37,869 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:51:37,874 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:51:37,876 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:51:37,883 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:51:37,884 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:51:37,889 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T07:51:37,895 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:51:37,903 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:51:37,905 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64540836, jitterRate=-0.03826659917831421}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T07:51:37,915 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731397897768Initializing all the Stores at 1731397897773 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397897774 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397897776 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397897777 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397897777Cleaning up temporary data from old regions at 1731397897884 (+107 ms)Region opened successfully at 1731397897915 (+31 ms) 2024-11-12T07:51:37,917 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T07:51:37,961 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1376fa8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f9fc96254ef8/172.17.0.2:0 2024-11-12T07:51:38,003 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T07:51:38,015 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T07:51:38,016 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T07:51:38,019 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T07:51:38,021 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T07:51:38,027 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T07:51:38,028 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T07:51:38,059 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T07:51:38,072 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T07:51:38,110 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T07:51:38,113 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T07:51:38,115 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T07:51:38,160 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T07:51:38,163 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T07:51:38,167 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T07:51:38,210 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T07:51:38,212 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T07:51:38,261 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T07:51:38,280 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T07:51:38,313 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T07:51:38,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T07:51:38,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T07:51:38,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:51:38,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:51:38,332 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f9fc96254ef8,42363,1731397895269, sessionid=0x1012f50d3b20000, setting cluster-up flag (Was=false) 2024-11-12T07:51:38,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:51:38,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:51:38,421 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T07:51:38,423 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f9fc96254ef8,42363,1731397895269 2024-11-12T07:51:38,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:51:38,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:51:38,512 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T07:51:38,514 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f9fc96254ef8,42363,1731397895269 2024-11-12T07:51:38,523 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T07:51:38,586 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(746): ClusterId : 5945df39-794d-4b95-a443-ecbda9762540 2024-11-12T07:51:38,589 DEBUG [RS:0;f9fc96254ef8:43507 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T07:51:38,612 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T07:51:38,622 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T07:51:38,623 DEBUG [RS:0;f9fc96254ef8:43507 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T07:51:38,624 DEBUG [RS:0;f9fc96254ef8:43507 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T07:51:38,629 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T07:51:38,639 DEBUG [RS:0;f9fc96254ef8:43507 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T07:51:38,640 DEBUG [RS:0;f9fc96254ef8:43507 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121444aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f9fc96254ef8/172.17.0.2:0 2024-11-12T07:51:38,637 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f9fc96254ef8,42363,1731397895269 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T07:51:38,648 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:51:38,649 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:51:38,649 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:51:38,649 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:51:38,649 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f9fc96254ef8:0, corePoolSize=10, maxPoolSize=10 2024-11-12T07:51:38,649 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:51:38,650 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-12T07:51:38,650 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:51:38,655 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731397928655 2024-11-12T07:51:38,656 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T07:51:38,656 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T07:51:38,657 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T07:51:38,659 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T07:51:38,660 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f9fc96254ef8:43507 2024-11-12T07:51:38,663 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:51:38,663 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T07:51:38,663 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T07:51:38,664 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T07:51:38,664 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T07:51:38,664 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T07:51:38,664 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T07:51:38,665 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T07:51:38,665 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T07:51:38,667 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T07:51:38,669 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(2659): reportForDuty to master=f9fc96254ef8,42363,1731397895269 with port=43507, startcode=1731397896042 2024-11-12T07:51:38,675 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T07:51:38,677 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T07:51:38,677 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T07:51:38,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741831_1007 (size=1321) 2024-11-12T07:51:38,681 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T07:51:38,682 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T07:51:38,684 DEBUG [RS:0;f9fc96254ef8:43507 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T07:51:38,686 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.large.0-1731397898683,5,FailOnTimeoutGroup] 2024-11-12T07:51:38,690 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.small.0-1731397898687,5,FailOnTimeoutGroup] 2024-11-12T07:51:38,691 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T07:51:38,691 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T07:51:38,692 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T07:51:38,692 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T07:51:38,696 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T07:51:38,696 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a 2024-11-12T07:51:38,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741832_1008 (size=32) 2024-11-12T07:51:38,723 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33755, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T07:51:38,729 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42363 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f9fc96254ef8,43507,1731397896042 2024-11-12T07:51:38,731 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42363 {}] master.ServerManager(517): Registering regionserver=f9fc96254ef8,43507,1731397896042 2024-11-12T07:51:38,744 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a 2024-11-12T07:51:38,745 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40941 2024-11-12T07:51:38,745 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T07:51:38,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T07:51:38,796 DEBUG [RS:0;f9fc96254ef8:43507 {}] zookeeper.ZKUtil(111): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f9fc96254ef8,43507,1731397896042 2024-11-12T07:51:38,796 WARN [RS:0;f9fc96254ef8:43507 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T07:51:38,796 INFO [RS:0;f9fc96254ef8:43507 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:51:38,796 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/WALs/f9fc96254ef8,43507,1731397896042 2024-11-12T07:51:38,798 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f9fc96254ef8,43507,1731397896042] 2024-11-12T07:51:38,820 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T07:51:38,836 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T07:51:38,841 INFO [RS:0;f9fc96254ef8:43507 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T07:51:38,841 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:51:38,842 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T07:51:38,847 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T07:51:38,848 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T07:51:38,849 DEBUG [RS:0;f9fc96254ef8:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:51:38,849 DEBUG [RS:0;f9fc96254ef8:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:51:38,849 DEBUG [RS:0;f9fc96254ef8:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:51:38,849 DEBUG [RS:0;f9fc96254ef8:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:51:38,849 DEBUG [RS:0;f9fc96254ef8:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:51:38,849 DEBUG [RS:0;f9fc96254ef8:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f9fc96254ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-12T07:51:38,850 DEBUG [RS:0;f9fc96254ef8:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:51:38,850 DEBUG [RS:0;f9fc96254ef8:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:51:38,850 DEBUG [RS:0;f9fc96254ef8:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:51:38,850 DEBUG [RS:0;f9fc96254ef8:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:51:38,850 DEBUG [RS:0;f9fc96254ef8:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:51:38,850 DEBUG [RS:0;f9fc96254ef8:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:51:38,850 DEBUG [RS:0;f9fc96254ef8:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f9fc96254ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-12T07:51:38,851 DEBUG [RS:0;f9fc96254ef8:43507 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-12T07:51:38,855 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T07:51:38,855 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T07:51:38,855 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:51:38,856 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T07:51:38,856 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T07:51:38,856 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43507,1731397896042-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T07:51:38,879 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T07:51:38,881 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43507,1731397896042-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:51:38,882 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:51:38,882 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.Replication(171): f9fc96254ef8,43507,1731397896042 started 2024-11-12T07:51:38,900 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:51:38,900 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1482): Serving as f9fc96254ef8,43507,1731397896042, RpcServer on f9fc96254ef8/172.17.0.2:43507, sessionid=0x1012f50d3b20001 2024-11-12T07:51:38,901 DEBUG [RS:0;f9fc96254ef8:43507 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T07:51:38,901 DEBUG [RS:0;f9fc96254ef8:43507 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f9fc96254ef8,43507,1731397896042 2024-11-12T07:51:38,901 DEBUG [RS:0;f9fc96254ef8:43507 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f9fc96254ef8,43507,1731397896042' 2024-11-12T07:51:38,902 DEBUG [RS:0;f9fc96254ef8:43507 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T07:51:38,903 DEBUG [RS:0;f9fc96254ef8:43507 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T07:51:38,904 DEBUG [RS:0;f9fc96254ef8:43507 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T07:51:38,904 DEBUG [RS:0;f9fc96254ef8:43507 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T07:51:38,904 DEBUG [RS:0;f9fc96254ef8:43507 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f9fc96254ef8,43507,1731397896042 2024-11-12T07:51:38,904 DEBUG [RS:0;f9fc96254ef8:43507 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f9fc96254ef8,43507,1731397896042' 2024-11-12T07:51:38,904 DEBUG [RS:0;f9fc96254ef8:43507 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T07:51:38,905 DEBUG [RS:0;f9fc96254ef8:43507 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T07:51:38,906 DEBUG [RS:0;f9fc96254ef8:43507 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T07:51:38,906 INFO [RS:0;f9fc96254ef8:43507 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T07:51:38,906 INFO [RS:0;f9fc96254ef8:43507 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T07:51:39,123 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:51:39,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T07:51:39,130 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T07:51:39,130 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:51:39,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:51:39,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T07:51:39,136 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T07:51:39,136 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:51:39,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:51:39,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T07:51:39,141 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T07:51:39,141 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:51:39,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:51:39,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T07:51:39,146 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T07:51:39,146 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:51:39,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:51:39,148 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T07:51:39,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740 2024-11-12T07:51:39,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740 2024-11-12T07:51:39,154 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T07:51:39,154 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T07:51:39,156 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T07:51:39,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T07:51:39,164 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:51:39,165 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73571819, jitterRate=0.09630553424358368}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T07:51:39,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731397899124Initializing all the Stores at 1731397899126 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397899126Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397899126Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397899126Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397899126Cleaning up temporary data from old regions at 1731397899155 (+29 ms)Region opened successfully at 1731397899169 (+14 ms) 2024-11-12T07:51:39,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T07:51:39,169 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T07:51:39,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T07:51:39,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T07:51:39,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T07:51:39,172 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T07:51:39,172 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731397899169Disabling compacts and flushes for region at 1731397899169Disabling writes for close at 1731397899170 (+1 ms)Writing region close event to WAL at 1731397899171 (+1 ms)Closed at 1731397899171 2024-11-12T07:51:39,175 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T07:51:39,176 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T07:51:39,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T07:51:39,190 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T07:51:39,195 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T07:51:39,346 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:40,347 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:41,347 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:42,348 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:43,349 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:44,351 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:45,352 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:45,612 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T07:51:45,613 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T07:51:46,352 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:47,353 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:48,354 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:49,354 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:50,355 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:51,356 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:52,357 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:53,357 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:54,358 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:55,359 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:56,360 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:57,361 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:58,361 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:51:59,362 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:52:00,363 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:52:01,364 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:52:02,364 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:52:03,365 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:52:04,287 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T07:52:04,366 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:52:05,367 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:52:06,367 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:52:07,368 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:52:08,369 WARN [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T07:52:08,911 INFO [RS:0;f9fc96254ef8:43507 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T07:52:08,915 INFO [RS:0;f9fc96254ef8:43507 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f9fc96254ef8%2C43507%2C1731397896042, suffix=, logDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/WALs/f9fc96254ef8,43507,1731397896042, archiveDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/oldWALs, maxLogs=32 2024-11-12T07:52:08,937 DEBUG [RS:0;f9fc96254ef8:43507 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/WALs/f9fc96254ef8,43507,1731397896042/f9fc96254ef8%2C43507%2C1731397896042.1731397928918, exclude list is [], retry=0 2024-11-12T07:52:08,941 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39471,DS-783c5011-81e9-4d06-b80e-db224bd39042,DISK] 2024-11-12T07:52:08,945 INFO [RS:0;f9fc96254ef8:43507 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/WALs/f9fc96254ef8,43507,1731397896042/f9fc96254ef8%2C43507%2C1731397896042.1731397928918 2024-11-12T07:52:08,946 DEBUG [RS:0;f9fc96254ef8:43507 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44047:44047)] 2024-11-12T07:52:09,122 DEBUG [f9fc96254ef8:42363 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T07:52:09,136 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:09,144 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f9fc96254ef8,43507,1731397896042, state=OPENING 2024-11-12T07:52:09,192 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T07:52:09,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:52:09,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:52:09,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:52:09,211 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:52:09,213 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T07:52:09,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f9fc96254ef8,43507,1731397896042}] 2024-11-12T07:52:09,399 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T07:52:09,403 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60769, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T07:52:09,414 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T07:52:09,414 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:52:09,415 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T07:52:09,418 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f9fc96254ef8%2C43507%2C1731397896042.meta, suffix=.meta, logDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/WALs/f9fc96254ef8,43507,1731397896042, archiveDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/oldWALs, maxLogs=32 2024-11-12T07:52:09,438 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/WALs/f9fc96254ef8,43507,1731397896042/f9fc96254ef8%2C43507%2C1731397896042.meta.1731397929421.meta, exclude list is [], retry=0 2024-11-12T07:52:09,442 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39471,DS-783c5011-81e9-4d06-b80e-db224bd39042,DISK] 2024-11-12T07:52:09,445 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/WALs/f9fc96254ef8,43507,1731397896042/f9fc96254ef8%2C43507%2C1731397896042.meta.1731397929421.meta 2024-11-12T07:52:09,446 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44047:44047)] 2024-11-12T07:52:09,446 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:52:09,448 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T07:52:09,451 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T07:52:09,457 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T07:52:09,462 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T07:52:09,463 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:09,463 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T07:52:09,463 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T07:52:09,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T07:52:09,468 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T07:52:09,468 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:09,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:52:09,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T07:52:09,471 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T07:52:09,471 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:09,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:52:09,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T07:52:09,474 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T07:52:09,474 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:09,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:52:09,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T07:52:09,478 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T07:52:09,478 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:09,479 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:52:09,479 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T07:52:09,481 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740 2024-11-12T07:52:09,484 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740 2024-11-12T07:52:09,486 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T07:52:09,486 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T07:52:09,487 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T07:52:09,490 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T07:52:09,492 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63964424, jitterRate=-0.046855807304382324}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T07:52:09,492 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T07:52:09,494 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731397929463Writing region info on filesystem at 1731397929464 (+1 ms)Initializing all the Stores at 1731397929465 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397929466 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397929466Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397929466Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397929466Cleaning up temporary data from old regions at 1731397929486 (+20 ms)Running coprocessor post-open hooks at 1731397929492 (+6 ms)Region opened successfully at 1731397929494 (+2 ms) 2024-11-12T07:52:09,502 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731397929392 2024-11-12T07:52:09,514 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T07:52:09,515 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T07:52:09,516 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:09,518 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f9fc96254ef8,43507,1731397896042, state=OPEN 2024-11-12T07:52:09,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:52:09,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:52:09,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:52:09,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:52:09,555 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:09,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T07:52:09,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f9fc96254ef8,43507,1731397896042 in 338 msec 2024-11-12T07:52:09,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T07:52:09,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.3820 sec 2024-11-12T07:52:09,571 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T07:52:09,571 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T07:52:09,592 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:52:09,593 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,43507,1731397896042, seqNum=-1] 2024-11-12T07:52:09,613 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:52:09,615 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44743, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:52:09,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 31.0780 sec 2024-11-12T07:52:09,637 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731397929637, completionTime=-1 2024-11-12T07:52:09,639 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-12T07:52:09,640 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T07:52:09,668 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-12T07:52:09,668 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731397989668 2024-11-12T07:52:09,669 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731398049669 2024-11-12T07:52:09,669 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-12T07:52:09,671 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,42363,1731397895269-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:52:09,671 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,42363,1731397895269-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:52:09,671 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,42363,1731397895269-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:52:09,673 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f9fc96254ef8:42363, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:52:09,673 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T07:52:09,675 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T07:52:09,680 DEBUG [master/f9fc96254ef8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T07:52:09,702 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.388sec 2024-11-12T07:52:09,704 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T07:52:09,706 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T07:52:09,707 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T07:52:09,708 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T07:52:09,708 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T07:52:09,708 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,42363,1731397895269-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T07:52:09,709 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,42363,1731397895269-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T07:52:09,716 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T07:52:09,717 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T07:52:09,717 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,42363,1731397895269-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:52:09,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@520ff257, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:52:09,729 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T07:52:09,729 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T07:52:09,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,42363,-1 for getting cluster id 2024-11-12T07:52:09,734 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:52:09,742 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5945df39-794d-4b95-a443-ecbda9762540' 2024-11-12T07:52:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:52:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5945df39-794d-4b95-a443-ecbda9762540" 2024-11-12T07:52:09,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18214de5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:52:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,42363,-1] 2024-11-12T07:52:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:52:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:52:09,755 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:52:09,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f263c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:52:09,759 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:52:09,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,43507,1731397896042, seqNum=-1] 2024-11-12T07:52:09,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:52:09,771 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:52:09,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f9fc96254ef8,42363,1731397895269 2024-11-12T07:52:09,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d1e1d56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:52:09,790 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,42363,-1 for getting cluster id 2024-11-12T07:52:09,791 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:52:09,792 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5945df39-794d-4b95-a443-ecbda9762540' 2024-11-12T07:52:09,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:52:09,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5945df39-794d-4b95-a443-ecbda9762540" 2024-11-12T07:52:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2adcb0ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:52:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,42363,-1] 2024-11-12T07:52:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:52:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:52:09,798 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:52:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a188c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:52:09,813 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=1142, ProcessCount=11, AvailableMemoryMB=7158 2024-11-12T07:52:09,820 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T07:52:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f9fc96254ef8,42363,1731397895269 2024-11-12T07:52:09,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@494ab692 2024-11-12T07:52:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T07:52:09,831 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T07:52:09,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:52:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-12T07:52:09,846 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:52:09,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-12T07:52:09,849 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:09,851 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:52:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:52:09,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741835_1011 (size=428) 2024-11-12T07:52:10,295 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c2da0bed673145aff0226f0ec2aa31e8, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a 2024-11-12T07:52:10,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741836_1012 (size=79) 2024-11-12T07:52:10,707 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:10,707 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing c2da0bed673145aff0226f0ec2aa31e8, disabling compactions & flushes 2024-11-12T07:52:10,708 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. 2024-11-12T07:52:10,708 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. 2024-11-12T07:52:10,708 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. after waiting 0 ms 2024-11-12T07:52:10,708 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. 2024-11-12T07:52:10,708 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. 2024-11-12T07:52:10,708 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for c2da0bed673145aff0226f0ec2aa31e8: Waiting for close lock at 1731397930707Disabling compacts and flushes for region at 1731397930707Disabling writes for close at 1731397930708 (+1 ms)Writing region close event to WAL at 1731397930708Closed at 1731397930708 2024-11-12T07:52:10,713 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:52:10,719 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731397930713"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397930713"}]},"ts":"1731397930713"} 2024-11-12T07:52:10,724 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:52:10,725 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:52:10,728 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397930726"}]},"ts":"1731397930726"} 2024-11-12T07:52:10,734 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-12T07:52:10,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=c2da0bed673145aff0226f0ec2aa31e8, ASSIGN}] 2024-11-12T07:52:10,739 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=c2da0bed673145aff0226f0ec2aa31e8, ASSIGN 2024-11-12T07:52:10,742 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=c2da0bed673145aff0226f0ec2aa31e8, ASSIGN; state=OFFLINE, location=f9fc96254ef8,43507,1731397896042; forceNewPlan=false, retain=false 2024-11-12T07:52:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:52:10,894 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c2da0bed673145aff0226f0ec2aa31e8, regionState=OPENING, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:10,902 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=c2da0bed673145aff0226f0ec2aa31e8, ASSIGN because future has completed 2024-11-12T07:52:10,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2da0bed673145aff0226f0ec2aa31e8, server=f9fc96254ef8,43507,1731397896042}] 2024-11-12T07:52:11,066 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. 2024-11-12T07:52:11,067 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c2da0bed673145aff0226f0ec2aa31e8, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:52:11,067 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync c2da0bed673145aff0226f0ec2aa31e8 2024-11-12T07:52:11,067 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:11,068 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c2da0bed673145aff0226f0ec2aa31e8 2024-11-12T07:52:11,068 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c2da0bed673145aff0226f0ec2aa31e8 2024-11-12T07:52:11,071 INFO [StoreOpener-c2da0bed673145aff0226f0ec2aa31e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c2da0bed673145aff0226f0ec2aa31e8 2024-11-12T07:52:11,074 INFO [StoreOpener-c2da0bed673145aff0226f0ec2aa31e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2da0bed673145aff0226f0ec2aa31e8 columnFamilyName testFamily 2024-11-12T07:52:11,074 DEBUG [StoreOpener-c2da0bed673145aff0226f0ec2aa31e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:11,075 INFO [StoreOpener-c2da0bed673145aff0226f0ec2aa31e8-1 {}] regionserver.HStore(327): Store=c2da0bed673145aff0226f0ec2aa31e8/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:52:11,075 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c2da0bed673145aff0226f0ec2aa31e8 2024-11-12T07:52:11,077 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c2da0bed673145aff0226f0ec2aa31e8 2024-11-12T07:52:11,078 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c2da0bed673145aff0226f0ec2aa31e8 2024-11-12T07:52:11,078 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c2da0bed673145aff0226f0ec2aa31e8 2024-11-12T07:52:11,078 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c2da0bed673145aff0226f0ec2aa31e8 2024-11-12T07:52:11,081 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c2da0bed673145aff0226f0ec2aa31e8 2024-11-12T07:52:11,085 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c2da0bed673145aff0226f0ec2aa31e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:52:11,086 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c2da0bed673145aff0226f0ec2aa31e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67679832, jitterRate=0.008508086204528809}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T07:52:11,087 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c2da0bed673145aff0226f0ec2aa31e8 2024-11-12T07:52:11,088 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c2da0bed673145aff0226f0ec2aa31e8: Running coprocessor pre-open hook at 1731397931068Writing region info on filesystem at 1731397931068Initializing all the Stores at 1731397931070 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397931070Cleaning up temporary data from old regions at 1731397931078 (+8 ms)Running coprocessor post-open hooks at 1731397931087 (+9 ms)Region opened successfully at 1731397931088 (+1 ms) 2024-11-12T07:52:11,090 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8., pid=6, masterSystemTime=1731397931058 2024-11-12T07:52:11,094 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. 2024-11-12T07:52:11,094 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. 2024-11-12T07:52:11,095 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c2da0bed673145aff0226f0ec2aa31e8, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:11,098 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2da0bed673145aff0226f0ec2aa31e8, server=f9fc96254ef8,43507,1731397896042 because future has completed 2024-11-12T07:52:11,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T07:52:11,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c2da0bed673145aff0226f0ec2aa31e8, server=f9fc96254ef8,43507,1731397896042 in 197 msec 2024-11-12T07:52:11,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T07:52:11,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=c2da0bed673145aff0226f0ec2aa31e8, ASSIGN in 370 msec 2024-11-12T07:52:11,113 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:52:11,114 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397931113"}]},"ts":"1731397931113"} 2024-11-12T07:52:11,117 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-12T07:52:11,119 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:52:11,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2800 sec 2024-11-12T07:52:12,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:52:12,896 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-12T07:52:12,896 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-12T07:52:12,898 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:12,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-12T07:52:12,906 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:12,907 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-12T07:52:12,910 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:52:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,43507,1731397896042, seqNum=-1] 2024-11-12T07:52:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:52:12,915 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:52:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2] 2024-11-12T07:52:12,935 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-12T07:52:12,938 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-12T07:52:12,943 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-12T07:52:12,947 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-12T07:52:12,949 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-12T07:52:12,950 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-7368373797696569336 2024-11-12T07:52:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:39384 deadline: 1731397935949, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-12T07:52:12,974 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-12T07:52:12,975 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-12T07:52:12,976 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-12T07:52:12,978 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-12T07:52:12,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-7368373797696569334 2024-11-12T07:52:12,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:39384 deadline: 1731397935978, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-12T07:52:12,983 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-12T07:52:12,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-7368373797696569333 2024-11-12T07:52:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:39384 deadline: 1731397935982, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-12T07:52:12,987 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-12T07:52:12,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-7368373797696569332 2024-11-12T07:52:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:39384 deadline: 1731397935987, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-12T07:52:12,992 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-12T07:52:13,003 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-398361687_22 at /127.0.0.1:37992 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1142 (was 1142), ProcessCount=11 (was 11), AvailableMemoryMB=7079 (was 7158) 2024-11-12T07:52:13,011 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=1142, ProcessCount=11, AvailableMemoryMB=7077 2024-11-12T07:52:13,015 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42363 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:52:13,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42363 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-12T07:52:13,020 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:52:13,020 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:13,020 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42363 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-12T07:52:13,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T07:52:13,022 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:52:13,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741837_1013 (size=414) 2024-11-12T07:52:13,436 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5be9542aae728d3ecaae7c77a6b7e6eb, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a 2024-11-12T07:52:13,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741838_1014 (size=65) 2024-11-12T07:52:13,847 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:13,847 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 5be9542aae728d3ecaae7c77a6b7e6eb, disabling compactions & flushes 2024-11-12T07:52:13,847 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. 2024-11-12T07:52:13,847 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. 2024-11-12T07:52:13,847 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. after waiting 0 ms 2024-11-12T07:52:13,847 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. 2024-11-12T07:52:13,847 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. 2024-11-12T07:52:13,848 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5be9542aae728d3ecaae7c77a6b7e6eb: Waiting for close lock at 1731397933847Disabling compacts and flushes for region at 1731397933847Disabling writes for close at 1731397933847Writing region close event to WAL at 1731397933847Closed at 1731397933847 2024-11-12T07:52:13,852 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:52:13,853 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1731397933852"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397933852"}]},"ts":"1731397933852"} 2024-11-12T07:52:13,857 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:52:13,860 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:52:13,861 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397933860"}]},"ts":"1731397933860"} 2024-11-12T07:52:13,864 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-12T07:52:13,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=5be9542aae728d3ecaae7c77a6b7e6eb, ASSIGN}] 2024-11-12T07:52:13,866 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=5be9542aae728d3ecaae7c77a6b7e6eb, ASSIGN 2024-11-12T07:52:13,868 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=5be9542aae728d3ecaae7c77a6b7e6eb, ASSIGN; state=OFFLINE, location=f9fc96254ef8,43507,1731397896042; forceNewPlan=false, retain=false 2024-11-12T07:52:14,019 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5be9542aae728d3ecaae7c77a6b7e6eb, regionState=OPENING, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:14,023 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=5be9542aae728d3ecaae7c77a6b7e6eb, ASSIGN because future has completed 2024-11-12T07:52:14,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5be9542aae728d3ecaae7c77a6b7e6eb, server=f9fc96254ef8,43507,1731397896042}] 2024-11-12T07:52:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T07:52:14,185 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. 2024-11-12T07:52:14,186 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 5be9542aae728d3ecaae7c77a6b7e6eb, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:52:14,186 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 5be9542aae728d3ecaae7c77a6b7e6eb 2024-11-12T07:52:14,186 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:14,186 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 5be9542aae728d3ecaae7c77a6b7e6eb 2024-11-12T07:52:14,186 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 5be9542aae728d3ecaae7c77a6b7e6eb 2024-11-12T07:52:14,189 INFO [StoreOpener-5be9542aae728d3ecaae7c77a6b7e6eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5be9542aae728d3ecaae7c77a6b7e6eb 2024-11-12T07:52:14,191 INFO [StoreOpener-5be9542aae728d3ecaae7c77a6b7e6eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5be9542aae728d3ecaae7c77a6b7e6eb columnFamilyName testFamily 2024-11-12T07:52:14,191 DEBUG [StoreOpener-5be9542aae728d3ecaae7c77a6b7e6eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:14,192 INFO [StoreOpener-5be9542aae728d3ecaae7c77a6b7e6eb-1 {}] regionserver.HStore(327): Store=5be9542aae728d3ecaae7c77a6b7e6eb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:52:14,192 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 5be9542aae728d3ecaae7c77a6b7e6eb 2024-11-12T07:52:14,193 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScanner/5be9542aae728d3ecaae7c77a6b7e6eb 2024-11-12T07:52:14,194 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScanner/5be9542aae728d3ecaae7c77a6b7e6eb 2024-11-12T07:52:14,195 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 5be9542aae728d3ecaae7c77a6b7e6eb 2024-11-12T07:52:14,195 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 5be9542aae728d3ecaae7c77a6b7e6eb 2024-11-12T07:52:14,198 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 5be9542aae728d3ecaae7c77a6b7e6eb 2024-11-12T07:52:14,201 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScanner/5be9542aae728d3ecaae7c77a6b7e6eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:52:14,202 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 5be9542aae728d3ecaae7c77a6b7e6eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73109265, jitterRate=0.08941294252872467}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T07:52:14,202 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5be9542aae728d3ecaae7c77a6b7e6eb 2024-11-12T07:52:14,203 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 5be9542aae728d3ecaae7c77a6b7e6eb: Running coprocessor pre-open hook at 1731397934186Writing region info on filesystem at 1731397934186Initializing all the Stores at 1731397934188 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397934188Cleaning up temporary data from old regions at 1731397934195 (+7 ms)Running coprocessor post-open hooks at 1731397934202 (+7 ms)Region opened successfully at 1731397934203 (+1 ms) 2024-11-12T07:52:14,205 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb., pid=9, masterSystemTime=1731397934177 2024-11-12T07:52:14,208 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. 2024-11-12T07:52:14,209 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. 2024-11-12T07:52:14,210 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5be9542aae728d3ecaae7c77a6b7e6eb, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:14,214 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5be9542aae728d3ecaae7c77a6b7e6eb, server=f9fc96254ef8,43507,1731397896042 because future has completed 2024-11-12T07:52:14,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-12T07:52:14,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 5be9542aae728d3ecaae7c77a6b7e6eb, server=f9fc96254ef8,43507,1731397896042 in 193 msec 2024-11-12T07:52:14,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T07:52:14,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=5be9542aae728d3ecaae7c77a6b7e6eb, ASSIGN in 356 msec 2024-11-12T07:52:14,226 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:52:14,226 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397934226"}]},"ts":"1731397934226"} 2024-11-12T07:52:14,229 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-12T07:52:14,231 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:52:14,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2160 sec 2024-11-12T07:52:14,999 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:52:15,020 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-12T07:52:15,021 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T07:52:15,022 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-12T07:52:15,612 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T07:52:15,612 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T07:52:15,613 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T07:52:15,613 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T07:52:15,614 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-12T07:52:15,614 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-12T07:52:15,615 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-12T07:52:15,615 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-12T07:52:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T07:52:16,065 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-12T07:52:16,065 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-12T07:52:16,066 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:16,074 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-12T07:52:16,074 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:16,075 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-12T07:52:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2] 2024-11-12T07:52:16,090 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-12T07:52:16,090 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-12T07:52:16,094 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-12T07:52:16,096 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-12T07:52:22,105 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=f9fc96254ef8:43507 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-11-12T07:52:22,106 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-11-12T07:52:22,106 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 from cache 2024-11-12T07:52:22,116 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1004 (was 1142), ProcessCount=11 (was 11), AvailableMemoryMB=6939 (was 7077) 2024-11-12T07:52:22,123 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=1004, ProcessCount=11, AvailableMemoryMB=6939 2024-11-12T07:52:22,126 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42363 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:52:22,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42363 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-12T07:52:22,129 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:52:22,130 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:22,130 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42363 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-12T07:52:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-12T07:52:22,131 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:52:22,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741839_1015 (size=412) 2024-11-12T07:52:22,550 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 02458ed77a017970a5c631399f75338a, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a 2024-11-12T07:52:22,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741840_1016 (size=63) 2024-11-12T07:52:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-12T07:52:22,965 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:22,965 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 02458ed77a017970a5c631399f75338a, disabling compactions & flushes 2024-11-12T07:52:22,965 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. 2024-11-12T07:52:22,965 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. 2024-11-12T07:52:22,965 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. after waiting 0 ms 2024-11-12T07:52:22,965 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. 2024-11-12T07:52:22,965 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. 2024-11-12T07:52:22,965 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 02458ed77a017970a5c631399f75338a: Waiting for close lock at 1731397942965Disabling compacts and flushes for region at 1731397942965Disabling writes for close at 1731397942965Writing region close event to WAL at 1731397942965Closed at 1731397942965 2024-11-12T07:52:22,967 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:52:22,967 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1731397942967"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397942967"}]},"ts":"1731397942967"} 2024-11-12T07:52:22,971 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:52:22,972 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:52:22,972 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397942972"}]},"ts":"1731397942972"} 2024-11-12T07:52:22,975 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-12T07:52:22,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=02458ed77a017970a5c631399f75338a, ASSIGN}] 2024-11-12T07:52:22,977 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=02458ed77a017970a5c631399f75338a, ASSIGN 2024-11-12T07:52:22,979 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=02458ed77a017970a5c631399f75338a, ASSIGN; state=OFFLINE, location=f9fc96254ef8,43507,1731397896042; forceNewPlan=false, retain=false 2024-11-12T07:52:23,130 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=02458ed77a017970a5c631399f75338a, regionState=OPENING, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:23,134 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=02458ed77a017970a5c631399f75338a, ASSIGN because future has completed 2024-11-12T07:52:23,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 02458ed77a017970a5c631399f75338a, server=f9fc96254ef8,43507,1731397896042}] 2024-11-12T07:52:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-12T07:52:23,293 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. 2024-11-12T07:52:23,294 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 02458ed77a017970a5c631399f75338a, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:52:23,294 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 02458ed77a017970a5c631399f75338a 2024-11-12T07:52:23,294 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:23,294 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 02458ed77a017970a5c631399f75338a 2024-11-12T07:52:23,294 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 02458ed77a017970a5c631399f75338a 2024-11-12T07:52:23,296 INFO [StoreOpener-02458ed77a017970a5c631399f75338a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 02458ed77a017970a5c631399f75338a 2024-11-12T07:52:23,298 INFO [StoreOpener-02458ed77a017970a5c631399f75338a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02458ed77a017970a5c631399f75338a columnFamilyName testFamily 2024-11-12T07:52:23,298 DEBUG [StoreOpener-02458ed77a017970a5c631399f75338a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:23,299 INFO [StoreOpener-02458ed77a017970a5c631399f75338a-1 {}] regionserver.HStore(327): Store=02458ed77a017970a5c631399f75338a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:52:23,299 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 02458ed77a017970a5c631399f75338a 2024-11-12T07:52:23,300 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNextAsync/02458ed77a017970a5c631399f75338a 2024-11-12T07:52:23,300 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNextAsync/02458ed77a017970a5c631399f75338a 2024-11-12T07:52:23,301 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 02458ed77a017970a5c631399f75338a 2024-11-12T07:52:23,301 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 02458ed77a017970a5c631399f75338a 2024-11-12T07:52:23,304 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 02458ed77a017970a5c631399f75338a 2024-11-12T07:52:23,309 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNextAsync/02458ed77a017970a5c631399f75338a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:52:23,310 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 02458ed77a017970a5c631399f75338a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58843949, jitterRate=-0.1231568306684494}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T07:52:23,311 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 02458ed77a017970a5c631399f75338a 2024-11-12T07:52:23,311 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 02458ed77a017970a5c631399f75338a: Running coprocessor pre-open hook at 1731397943294Writing region info on filesystem at 1731397943294Initializing all the Stores at 1731397943296 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397943296Cleaning up temporary data from old regions at 1731397943301 (+5 ms)Running coprocessor post-open hooks at 1731397943311 (+10 ms)Region opened successfully at 1731397943311 2024-11-12T07:52:23,313 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a., pid=12, masterSystemTime=1731397943288 2024-11-12T07:52:23,316 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. 2024-11-12T07:52:23,316 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. 2024-11-12T07:52:23,317 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=02458ed77a017970a5c631399f75338a, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:23,320 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 02458ed77a017970a5c631399f75338a, server=f9fc96254ef8,43507,1731397896042 because future has completed 2024-11-12T07:52:23,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-12T07:52:23,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 02458ed77a017970a5c631399f75338a, server=f9fc96254ef8,43507,1731397896042 in 188 msec 2024-11-12T07:52:23,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-12T07:52:23,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=02458ed77a017970a5c631399f75338a, ASSIGN in 353 msec 2024-11-12T07:52:23,332 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:52:23,332 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397943332"}]},"ts":"1731397943332"} 2024-11-12T07:52:23,335 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-12T07:52:23,337 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:52:23,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2110 sec 2024-11-12T07:52:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-12T07:52:25,165 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-12T07:52:25,165 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-12T07:52:25,165 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:25,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-12T07:52:25,173 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:25,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-12T07:52:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2] 2024-11-12T07:52:25,190 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-12T07:52:25,190 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-12T07:52:25,194 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-12T07:52:25,196 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-12T07:52:25,198 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-12T07:52:25,199 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-12T07:52:25,612 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-12T07:52:25,612 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-12T07:52:27,025 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-12T07:52:28,834 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:52:31,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43507 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7368373797696569323' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-12T07:52:31,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43507 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:39384 deadline: 1731397957205, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7368373797696569323' does not exist 2024-11-12T07:52:31,213 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to f9fc96254ef8,43507,1731397896042 for closing scanner id = -7368373797696569323 for 02458ed77a017970a5c631399f75338a of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7368373797696569323' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:52:31,216 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=888 (was 1004), ProcessCount=11 (was 11), AvailableMemoryMB=6807 (was 6939) 2024-11-12T07:52:31,225 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=888, ProcessCount=11, AvailableMemoryMB=6807 2024-11-12T07:52:31,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:52:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-12T07:52:31,231 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:52:31,231 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:31,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-12T07:52:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T07:52:31,232 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:52:31,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741841_1017 (size=423) 2024-11-12T07:52:31,650 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5e6fa6f9df49a0f74aee6be9e175a9d5, NAME => 'testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a 2024-11-12T07:52:31,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741842_1018 (size=74) 2024-11-12T07:52:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-12T07:52:32,063 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:32,064 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 5e6fa6f9df49a0f74aee6be9e175a9d5, disabling compactions & flushes 2024-11-12T07:52:32,064 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. 2024-11-12T07:52:32,064 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. 2024-11-12T07:52:32,064 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. after waiting 0 ms 2024-11-12T07:52:32,065 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. 2024-11-12T07:52:32,065 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. 2024-11-12T07:52:32,065 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5e6fa6f9df49a0f74aee6be9e175a9d5: Waiting for close lock at 1731397952064Disabling compacts and flushes for region at 1731397952064Disabling writes for close at 1731397952064Writing region close event to WAL at 1731397952065 (+1 ms)Closed at 1731397952065 2024-11-12T07:52:32,068 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:52:32,069 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731397952068"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397952068"}]},"ts":"1731397952068"} 2024-11-12T07:52:32,073 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:52:32,075 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:52:32,076 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397952075"}]},"ts":"1731397952075"} 2024-11-12T07:52:32,079 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-12T07:52:32,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5e6fa6f9df49a0f74aee6be9e175a9d5, ASSIGN}] 2024-11-12T07:52:32,082 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5e6fa6f9df49a0f74aee6be9e175a9d5, ASSIGN 2024-11-12T07:52:32,084 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5e6fa6f9df49a0f74aee6be9e175a9d5, ASSIGN; state=OFFLINE, location=f9fc96254ef8,43507,1731397896042; forceNewPlan=false, retain=false 2024-11-12T07:52:32,235 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5e6fa6f9df49a0f74aee6be9e175a9d5, regionState=OPENING, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:32,243 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5e6fa6f9df49a0f74aee6be9e175a9d5, ASSIGN because future has completed 2024-11-12T07:52:32,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e6fa6f9df49a0f74aee6be9e175a9d5, server=f9fc96254ef8,43507,1731397896042}] 2024-11-12T07:52:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T07:52:32,405 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. 2024-11-12T07:52:32,405 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e6fa6f9df49a0f74aee6be9e175a9d5, NAME => 'testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:52:32,405 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 5e6fa6f9df49a0f74aee6be9e175a9d5 2024-11-12T07:52:32,405 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:32,406 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 5e6fa6f9df49a0f74aee6be9e175a9d5 2024-11-12T07:52:32,406 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 5e6fa6f9df49a0f74aee6be9e175a9d5 2024-11-12T07:52:32,408 INFO [StoreOpener-5e6fa6f9df49a0f74aee6be9e175a9d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5e6fa6f9df49a0f74aee6be9e175a9d5 2024-11-12T07:52:32,411 INFO [StoreOpener-5e6fa6f9df49a0f74aee6be9e175a9d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e6fa6f9df49a0f74aee6be9e175a9d5 columnFamilyName testFamily 2024-11-12T07:52:32,411 DEBUG [StoreOpener-5e6fa6f9df49a0f74aee6be9e175a9d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:32,412 INFO [StoreOpener-5e6fa6f9df49a0f74aee6be9e175a9d5-1 {}] regionserver.HStore(327): Store=5e6fa6f9df49a0f74aee6be9e175a9d5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:52:32,412 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 5e6fa6f9df49a0f74aee6be9e175a9d5 2024-11-12T07:52:32,414 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextException/5e6fa6f9df49a0f74aee6be9e175a9d5 2024-11-12T07:52:32,414 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextException/5e6fa6f9df49a0f74aee6be9e175a9d5 2024-11-12T07:52:32,415 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 5e6fa6f9df49a0f74aee6be9e175a9d5 2024-11-12T07:52:32,415 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 5e6fa6f9df49a0f74aee6be9e175a9d5 2024-11-12T07:52:32,418 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 5e6fa6f9df49a0f74aee6be9e175a9d5 2024-11-12T07:52:32,421 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextException/5e6fa6f9df49a0f74aee6be9e175a9d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:52:32,422 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 5e6fa6f9df49a0f74aee6be9e175a9d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59991105, jitterRate=-0.1060628741979599}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T07:52:32,423 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e6fa6f9df49a0f74aee6be9e175a9d5 2024-11-12T07:52:32,423 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 5e6fa6f9df49a0f74aee6be9e175a9d5: Running coprocessor pre-open hook at 1731397952406Writing region info on filesystem at 1731397952406Initializing all the Stores at 1731397952407 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397952408 (+1 ms)Cleaning up temporary data from old regions at 1731397952415 (+7 ms)Running coprocessor post-open hooks at 1731397952423 (+8 ms)Region opened successfully at 1731397952423 2024-11-12T07:52:32,425 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5., pid=15, masterSystemTime=1731397952399 2024-11-12T07:52:32,428 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. 2024-11-12T07:52:32,428 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. 2024-11-12T07:52:32,429 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5e6fa6f9df49a0f74aee6be9e175a9d5, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:32,432 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e6fa6f9df49a0f74aee6be9e175a9d5, server=f9fc96254ef8,43507,1731397896042 because future has completed 2024-11-12T07:52:32,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-12T07:52:32,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e6fa6f9df49a0f74aee6be9e175a9d5, server=f9fc96254ef8,43507,1731397896042 in 190 msec 2024-11-12T07:52:32,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-12T07:52:32,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5e6fa6f9df49a0f74aee6be9e175a9d5, ASSIGN in 357 msec 2024-11-12T07:52:32,442 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:52:32,442 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397952442"}]},"ts":"1731397952442"} 2024-11-12T07:52:32,445 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-12T07:52:32,446 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:52:32,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2190 sec 2024-11-12T07:52:33,027 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-12T07:52:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T07:52:34,255 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-12T07:52:34,255 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-12T07:52:34,255 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:34,259 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-12T07:52:34,260 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:34,260 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-12T07:52:34,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2] 2024-11-12T07:52:34,276 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-12T07:52:34,276 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-12T07:52:34,279 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-12T07:52:34,282 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-12T07:52:34,283 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-12T07:52:34,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-7368373797696569319 2024-11-12T07:52:34,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:39384 deadline: 1731397957283, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-12T07:52:34,287 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T07:52:34,287 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-12T07:52:34,289 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-12T07:52:34,289 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-12T07:52:34,291 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-12T07:52:34,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-7368373797696569317 2024-11-12T07:52:34,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:39384 deadline: 1731397957290, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-12T07:52:34,295 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-12T07:52:34,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-7368373797696569316 2024-11-12T07:52:34,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:39384 deadline: 1731397957294, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-12T07:52:34,299 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-12T07:52:34,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-7368373797696569315 2024-11-12T07:52:34,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:39384 deadline: 1731397957298, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-12T07:52:34,303 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-12T07:52:34,311 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 225), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=817 (was 888), ProcessCount=11 (was 11), AvailableMemoryMB=6718 (was 6807) 2024-11-12T07:52:34,318 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=817, ProcessCount=11, AvailableMemoryMB=6718 2024-11-12T07:52:34,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:52:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-12T07:52:34,324 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:52:34,325 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:34,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-12T07:52:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-12T07:52:34,326 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:52:34,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741843_1019 (size=418) 2024-11-12T07:52:34,741 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => de9251dafa4bdb68ff37f1d9fec5985a, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a 2024-11-12T07:52:34,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741844_1020 (size=69) 2024-11-12T07:52:35,152 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:35,152 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing de9251dafa4bdb68ff37f1d9fec5985a, disabling compactions & flushes 2024-11-12T07:52:35,153 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. 2024-11-12T07:52:35,153 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. 2024-11-12T07:52:35,153 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. after waiting 0 ms 2024-11-12T07:52:35,153 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. 2024-11-12T07:52:35,153 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. 2024-11-12T07:52:35,153 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for de9251dafa4bdb68ff37f1d9fec5985a: Waiting for close lock at 1731397955152Disabling compacts and flushes for region at 1731397955152Disabling writes for close at 1731397955153 (+1 ms)Writing region close event to WAL at 1731397955153Closed at 1731397955153 2024-11-12T07:52:35,155 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:52:35,155 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1731397955155"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397955155"}]},"ts":"1731397955155"} 2024-11-12T07:52:35,158 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:52:35,159 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:52:35,160 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397955159"}]},"ts":"1731397955159"} 2024-11-12T07:52:35,162 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-12T07:52:35,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=de9251dafa4bdb68ff37f1d9fec5985a, ASSIGN}] 2024-11-12T07:52:35,164 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=de9251dafa4bdb68ff37f1d9fec5985a, ASSIGN 2024-11-12T07:52:35,166 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=de9251dafa4bdb68ff37f1d9fec5985a, ASSIGN; state=OFFLINE, location=f9fc96254ef8,43507,1731397896042; forceNewPlan=false, retain=false 2024-11-12T07:52:35,317 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=de9251dafa4bdb68ff37f1d9fec5985a, regionState=OPENING, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:35,321 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=de9251dafa4bdb68ff37f1d9fec5985a, ASSIGN because future has completed 2024-11-12T07:52:35,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9251dafa4bdb68ff37f1d9fec5985a, server=f9fc96254ef8,43507,1731397896042}] 2024-11-12T07:52:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-12T07:52:35,482 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. 2024-11-12T07:52:35,482 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => de9251dafa4bdb68ff37f1d9fec5985a, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:52:35,483 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:52:35,483 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:35,483 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:52:35,483 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:52:35,485 INFO [StoreOpener-de9251dafa4bdb68ff37f1d9fec5985a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:52:35,487 INFO [StoreOpener-de9251dafa4bdb68ff37f1d9fec5985a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de9251dafa4bdb68ff37f1d9fec5985a columnFamilyName testFamily 2024-11-12T07:52:35,488 DEBUG [StoreOpener-de9251dafa4bdb68ff37f1d9fec5985a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:35,488 INFO [StoreOpener-de9251dafa4bdb68ff37f1d9fec5985a-1 {}] regionserver.HStore(327): Store=de9251dafa4bdb68ff37f1d9fec5985a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:52:35,488 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:52:35,490 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScanner/de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:52:35,490 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScanner/de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:52:35,491 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:52:35,491 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:52:35,494 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:52:35,497 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScanner/de9251dafa4bdb68ff37f1d9fec5985a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:52:35,498 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened de9251dafa4bdb68ff37f1d9fec5985a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62151478, jitterRate=-0.07387080788612366}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T07:52:35,498 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:52:35,498 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for de9251dafa4bdb68ff37f1d9fec5985a: Running coprocessor pre-open hook at 1731397955483Writing region info on filesystem at 1731397955483Initializing all the Stores at 1731397955485 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397955485Cleaning up temporary data from old regions at 1731397955491 (+6 ms)Running coprocessor post-open hooks at 1731397955498 (+7 ms)Region opened successfully at 1731397955498 2024-11-12T07:52:35,500 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a., pid=18, masterSystemTime=1731397955476 2024-11-12T07:52:35,502 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. 2024-11-12T07:52:35,502 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. 2024-11-12T07:52:35,503 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=de9251dafa4bdb68ff37f1d9fec5985a, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:35,505 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9251dafa4bdb68ff37f1d9fec5985a, server=f9fc96254ef8,43507,1731397896042 because future has completed 2024-11-12T07:52:35,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-12T07:52:35,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure de9251dafa4bdb68ff37f1d9fec5985a, server=f9fc96254ef8,43507,1731397896042 in 185 msec 2024-11-12T07:52:35,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-12T07:52:35,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=de9251dafa4bdb68ff37f1d9fec5985a, ASSIGN in 347 msec 2024-11-12T07:52:35,514 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:52:35,514 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397955514"}]},"ts":"1731397955514"} 2024-11-12T07:52:35,517 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-12T07:52:35,518 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:52:35,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.1980 sec 2024-11-12T07:52:35,612 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-12T07:52:35,612 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-12T07:52:35,615 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-12T07:52:35,615 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-12T07:52:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-12T07:52:37,347 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-12T07:52:37,347 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-12T07:52:37,348 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:37,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-12T07:52:37,357 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:37,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-12T07:52:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2] 2024-11-12T07:52:37,371 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-12T07:52:37,371 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-12T07:52:37,374 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-12T07:52:37,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-12T07:52:37,910 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:52:38,378 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=f9fc96254ef8:43507 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-12T07:52:38,379 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-12T07:52:38,379 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 from cache 2024-11-12T07:52:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-12T07:52:39,029 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-12T07:52:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2] 2024-11-12T07:52:39,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-12T07:52:40,378 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=f9fc96254ef8:43507 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=986ms, rpcTimeout=984ms 2024-11-12T07:52:40,378 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=986ms, rpcTimeout=984ms 2024-11-12T07:52:40,378 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 from cache 2024-11-12T07:52:40,388 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=767 (was 817), ProcessCount=11 (was 11), AvailableMemoryMB=6710 (was 6718) 2024-11-12T07:52:40,398 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=767, ProcessCount=11, AvailableMemoryMB=6710 2024-11-12T07:52:40,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:52:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-12T07:52:40,405 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:52:40,405 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:40,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-12T07:52:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T07:52:40,407 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:52:40,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741845_1021 (size=416) 2024-11-12T07:52:40,825 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d1b2734401ee4bc63a4d1307e3657981, NAME => 'testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a 2024-11-12T07:52:40,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741846_1022 (size=67) 2024-11-12T07:52:40,841 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:40,841 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing d1b2734401ee4bc63a4d1307e3657981, disabling compactions & flushes 2024-11-12T07:52:40,841 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. 2024-11-12T07:52:40,841 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. 2024-11-12T07:52:40,842 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. after waiting 0 ms 2024-11-12T07:52:40,842 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. 2024-11-12T07:52:40,842 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. 2024-11-12T07:52:40,842 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for d1b2734401ee4bc63a4d1307e3657981: Waiting for close lock at 1731397960841Disabling compacts and flushes for region at 1731397960841Disabling writes for close at 1731397960842 (+1 ms)Writing region close event to WAL at 1731397960842Closed at 1731397960842 2024-11-12T07:52:40,845 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:52:40,846 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731397960845"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397960845"}]},"ts":"1731397960845"} 2024-11-12T07:52:40,849 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:52:40,851 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:52:40,852 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397960851"}]},"ts":"1731397960851"} 2024-11-12T07:52:40,855 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-12T07:52:40,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=d1b2734401ee4bc63a4d1307e3657981, ASSIGN}] 2024-11-12T07:52:40,857 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=d1b2734401ee4bc63a4d1307e3657981, ASSIGN 2024-11-12T07:52:40,859 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=d1b2734401ee4bc63a4d1307e3657981, ASSIGN; state=OFFLINE, location=f9fc96254ef8,43507,1731397896042; forceNewPlan=false, retain=false 2024-11-12T07:52:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-12T07:52:41,010 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d1b2734401ee4bc63a4d1307e3657981, regionState=OPENING, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:41,016 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=d1b2734401ee4bc63a4d1307e3657981, ASSIGN because future has completed 2024-11-12T07:52:41,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1b2734401ee4bc63a4d1307e3657981, server=f9fc96254ef8,43507,1731397896042}] 2024-11-12T07:52:41,181 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. 2024-11-12T07:52:41,182 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => d1b2734401ee4bc63a4d1307e3657981, NAME => 'testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:52:41,183 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync d1b2734401ee4bc63a4d1307e3657981 2024-11-12T07:52:41,183 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:41,183 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for d1b2734401ee4bc63a4d1307e3657981 2024-11-12T07:52:41,183 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for d1b2734401ee4bc63a4d1307e3657981 2024-11-12T07:52:41,186 INFO [StoreOpener-d1b2734401ee4bc63a4d1307e3657981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d1b2734401ee4bc63a4d1307e3657981 2024-11-12T07:52:41,189 INFO [StoreOpener-d1b2734401ee4bc63a4d1307e3657981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1b2734401ee4bc63a4d1307e3657981 columnFamilyName testFamily 2024-11-12T07:52:41,189 DEBUG [StoreOpener-d1b2734401ee4bc63a4d1307e3657981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:41,190 INFO [StoreOpener-d1b2734401ee4bc63a4d1307e3657981-1 {}] regionserver.HStore(327): Store=d1b2734401ee4bc63a4d1307e3657981/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:52:41,190 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for d1b2734401ee4bc63a4d1307e3657981 2024-11-12T07:52:41,191 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNextAsync/d1b2734401ee4bc63a4d1307e3657981 2024-11-12T07:52:41,192 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNextAsync/d1b2734401ee4bc63a4d1307e3657981 2024-11-12T07:52:41,192 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for d1b2734401ee4bc63a4d1307e3657981 2024-11-12T07:52:41,193 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for d1b2734401ee4bc63a4d1307e3657981 2024-11-12T07:52:41,195 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for d1b2734401ee4bc63a4d1307e3657981 2024-11-12T07:52:41,198 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNextAsync/d1b2734401ee4bc63a4d1307e3657981/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:52:41,198 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened d1b2734401ee4bc63a4d1307e3657981; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71120643, jitterRate=0.05978016555309296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T07:52:41,198 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1b2734401ee4bc63a4d1307e3657981 2024-11-12T07:52:41,199 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for d1b2734401ee4bc63a4d1307e3657981: Running coprocessor pre-open hook at 1731397961183Writing region info on filesystem at 1731397961183Initializing all the Stores at 1731397961185 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397961186 (+1 ms)Cleaning up temporary data from old regions at 1731397961193 (+7 ms)Running coprocessor post-open hooks at 1731397961198 (+5 ms)Region opened successfully at 1731397961199 (+1 ms) 2024-11-12T07:52:41,200 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981., pid=21, masterSystemTime=1731397961171 2024-11-12T07:52:41,202 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. 2024-11-12T07:52:41,203 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. 2024-11-12T07:52:41,203 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d1b2734401ee4bc63a4d1307e3657981, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:41,206 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1b2734401ee4bc63a4d1307e3657981, server=f9fc96254ef8,43507,1731397896042 because future has completed 2024-11-12T07:52:41,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-12T07:52:41,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure d1b2734401ee4bc63a4d1307e3657981, server=f9fc96254ef8,43507,1731397896042 in 190 msec 2024-11-12T07:52:41,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-12T07:52:41,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=d1b2734401ee4bc63a4d1307e3657981, ASSIGN in 354 msec 2024-11-12T07:52:41,213 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:52:41,213 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397961213"}]},"ts":"1731397961213"} 2024-11-12T07:52:41,216 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-12T07:52:41,217 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:52:41,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 816 msec 2024-11-12T07:52:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T07:52:41,425 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-12T07:52:41,425 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-12T07:52:41,425 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:41,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-12T07:52:41,431 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:41,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-12T07:52:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2] 2024-11-12T07:52:41,446 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-12T07:52:41,446 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-12T07:52:41,449 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-12T07:52:41,452 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-12T07:52:41,453 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-12T07:52:41,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-12T07:52:44,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43507 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7368373797696569304' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-12T07:52:44,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43507 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:39384 deadline: 1731397965459, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7368373797696569304' does not exist 2024-11-12T07:52:44,472 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to f9fc96254ef8,43507,1731397896042 for closing scanner id = -7368373797696569304 for d1b2734401ee4bc63a4d1307e3657981 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7368373797696569304' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:52:44,476 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-398361687_22 at /127.0.0.1:59522 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=706 (was 767), ProcessCount=11 (was 11), AvailableMemoryMB=6701 (was 6710) 2024-11-12T07:52:44,484 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=706, ProcessCount=11, AvailableMemoryMB=6700 2024-11-12T07:52:44,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:52:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-12T07:52:44,490 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:52:44,490 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:44,490 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-12T07:52:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-12T07:52:44,491 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:52:44,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741847_1023 (size=411) 2024-11-12T07:52:44,902 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 26d0ebdc343ac286c84ab8127870729b, NAME => 'testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a 2024-11-12T07:52:44,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741848_1024 (size=62) 2024-11-12T07:52:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-12T07:52:45,032 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-12T07:52:45,314 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:45,315 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 26d0ebdc343ac286c84ab8127870729b, disabling compactions & flushes 2024-11-12T07:52:45,315 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. 2024-11-12T07:52:45,315 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. 2024-11-12T07:52:45,315 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. after waiting 0 ms 2024-11-12T07:52:45,315 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. 2024-11-12T07:52:45,316 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. 2024-11-12T07:52:45,316 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 26d0ebdc343ac286c84ab8127870729b: Waiting for close lock at 1731397965315Disabling compacts and flushes for region at 1731397965315Disabling writes for close at 1731397965315Writing region close event to WAL at 1731397965316 (+1 ms)Closed at 1731397965316 2024-11-12T07:52:45,319 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:52:45,319 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731397965319"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397965319"}]},"ts":"1731397965319"} 2024-11-12T07:52:45,322 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:52:45,324 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:52:45,324 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397965324"}]},"ts":"1731397965324"} 2024-11-12T07:52:45,327 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-12T07:52:45,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=26d0ebdc343ac286c84ab8127870729b, ASSIGN}] 2024-11-12T07:52:45,329 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=26d0ebdc343ac286c84ab8127870729b, ASSIGN 2024-11-12T07:52:45,330 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=26d0ebdc343ac286c84ab8127870729b, ASSIGN; state=OFFLINE, location=f9fc96254ef8,43507,1731397896042; forceNewPlan=false, retain=false 2024-11-12T07:52:45,481 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=26d0ebdc343ac286c84ab8127870729b, regionState=OPENING, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:45,485 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=26d0ebdc343ac286c84ab8127870729b, ASSIGN because future has completed 2024-11-12T07:52:45,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26d0ebdc343ac286c84ab8127870729b, server=f9fc96254ef8,43507,1731397896042}] 2024-11-12T07:52:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-12T07:52:45,612 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-12T07:52:45,612 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-12T07:52:45,645 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. 2024-11-12T07:52:45,645 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 26d0ebdc343ac286c84ab8127870729b, NAME => 'testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:52:45,645 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 26d0ebdc343ac286c84ab8127870729b 2024-11-12T07:52:45,645 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:45,646 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 26d0ebdc343ac286c84ab8127870729b 2024-11-12T07:52:45,646 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 26d0ebdc343ac286c84ab8127870729b 2024-11-12T07:52:45,647 INFO [StoreOpener-26d0ebdc343ac286c84ab8127870729b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 26d0ebdc343ac286c84ab8127870729b 2024-11-12T07:52:45,650 INFO [StoreOpener-26d0ebdc343ac286c84ab8127870729b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 26d0ebdc343ac286c84ab8127870729b columnFamilyName testFamily 2024-11-12T07:52:45,650 DEBUG [StoreOpener-26d0ebdc343ac286c84ab8127870729b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:45,650 INFO [StoreOpener-26d0ebdc343ac286c84ab8127870729b-1 {}] regionserver.HStore(327): Store=26d0ebdc343ac286c84ab8127870729b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:52:45,651 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 26d0ebdc343ac286c84ab8127870729b 2024-11-12T07:52:45,652 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNext/26d0ebdc343ac286c84ab8127870729b 2024-11-12T07:52:45,652 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNext/26d0ebdc343ac286c84ab8127870729b 2024-11-12T07:52:45,653 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 26d0ebdc343ac286c84ab8127870729b 2024-11-12T07:52:45,653 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 26d0ebdc343ac286c84ab8127870729b 2024-11-12T07:52:45,656 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 26d0ebdc343ac286c84ab8127870729b 2024-11-12T07:52:45,659 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNext/26d0ebdc343ac286c84ab8127870729b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:52:45,659 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 26d0ebdc343ac286c84ab8127870729b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65044931, jitterRate=-0.030754998326301575}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T07:52:45,660 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 26d0ebdc343ac286c84ab8127870729b 2024-11-12T07:52:45,660 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 26d0ebdc343ac286c84ab8127870729b: Running coprocessor pre-open hook at 1731397965646Writing region info on filesystem at 1731397965646Initializing all the Stores at 1731397965647 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397965647Cleaning up temporary data from old regions at 1731397965653 (+6 ms)Running coprocessor post-open hooks at 1731397965660 (+7 ms)Region opened successfully at 1731397965660 2024-11-12T07:52:45,661 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b., pid=24, masterSystemTime=1731397965640 2024-11-12T07:52:45,664 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. 2024-11-12T07:52:45,664 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. 2024-11-12T07:52:45,665 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=26d0ebdc343ac286c84ab8127870729b, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:45,667 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 26d0ebdc343ac286c84ab8127870729b, server=f9fc96254ef8,43507,1731397896042 because future has completed 2024-11-12T07:52:45,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-12T07:52:45,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 26d0ebdc343ac286c84ab8127870729b, server=f9fc96254ef8,43507,1731397896042 in 182 msec 2024-11-12T07:52:45,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-12T07:52:45,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=26d0ebdc343ac286c84ab8127870729b, ASSIGN in 344 msec 2024-11-12T07:52:45,675 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:52:45,675 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397965675"}]},"ts":"1731397965675"} 2024-11-12T07:52:45,677 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-12T07:52:45,678 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:52:45,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.1910 sec 2024-11-12T07:52:46,689 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:52:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-12T07:52:47,517 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-12T07:52:47,517 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-12T07:52:47,518 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:47,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-12T07:52:47,529 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:47,530 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-12T07:52:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2] 2024-11-12T07:52:47,544 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-12T07:52:47,544 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-12T07:52:47,547 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-12T07:52:47,549 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-12T07:52:47,551 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-12T07:52:47,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-12T07:52:49,743 INFO [master/f9fc96254ef8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-12T07:52:49,743 INFO [master/f9fc96254ef8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-12T07:52:50,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43507 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7368373797696569300' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-12T07:52:50,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43507 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:39384 deadline: 1731397971563, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7368373797696569300' does not exist 2024-11-12T07:52:50,564 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to f9fc96254ef8,43507,1731397896042 for closing scanner id = -7368373797696569300 for 26d0ebdc343ac286c84ab8127870729b of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7368373797696569300' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:52:50,570 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 223), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=649 (was 706), ProcessCount=11 (was 11), AvailableMemoryMB=6694 (was 6700) 2024-11-12T07:52:50,580 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=649, ProcessCount=11, AvailableMemoryMB=6694 2024-11-12T07:52:50,582 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42363 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:52:50,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42363 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-12T07:52:50,586 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:52:50,586 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:50,587 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42363 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-12T07:52:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T07:52:50,588 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:52:50,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741849_1025 (size=419) 2024-11-12T07:52:51,004 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 96b172c55f3e7fec396050c330fc9f7b, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a 2024-11-12T07:52:51,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741850_1026 (size=70) 2024-11-12T07:52:51,036 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-12T07:52:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-12T07:52:51,419 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:51,420 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 96b172c55f3e7fec396050c330fc9f7b, disabling compactions & flushes 2024-11-12T07:52:51,420 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. 2024-11-12T07:52:51,420 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. 2024-11-12T07:52:51,420 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. after waiting 0 ms 2024-11-12T07:52:51,420 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. 2024-11-12T07:52:51,421 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. 2024-11-12T07:52:51,421 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 96b172c55f3e7fec396050c330fc9f7b: Waiting for close lock at 1731397971419Disabling compacts and flushes for region at 1731397971419Disabling writes for close at 1731397971420 (+1 ms)Writing region close event to WAL at 1731397971420Closed at 1731397971420 2024-11-12T07:52:51,427 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:52:51,427 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731397971427"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397971427"}]},"ts":"1731397971427"} 2024-11-12T07:52:51,430 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:52:51,432 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:52:51,432 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397971432"}]},"ts":"1731397971432"} 2024-11-12T07:52:51,434 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-12T07:52:51,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=96b172c55f3e7fec396050c330fc9f7b, ASSIGN}] 2024-11-12T07:52:51,436 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=96b172c55f3e7fec396050c330fc9f7b, ASSIGN 2024-11-12T07:52:51,437 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=96b172c55f3e7fec396050c330fc9f7b, ASSIGN; state=OFFLINE, location=f9fc96254ef8,43507,1731397896042; forceNewPlan=false, retain=false 2024-11-12T07:52:51,588 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=96b172c55f3e7fec396050c330fc9f7b, regionState=OPENING, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:51,595 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=96b172c55f3e7fec396050c330fc9f7b, ASSIGN because future has completed 2024-11-12T07:52:51,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96b172c55f3e7fec396050c330fc9f7b, server=f9fc96254ef8,43507,1731397896042}] 2024-11-12T07:52:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T07:52:51,762 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. 2024-11-12T07:52:51,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 96b172c55f3e7fec396050c330fc9f7b, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:52:51,763 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:52:51,763 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:52:51,763 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:52:51,763 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:52:51,765 INFO [StoreOpener-96b172c55f3e7fec396050c330fc9f7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:52:51,767 INFO [StoreOpener-96b172c55f3e7fec396050c330fc9f7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96b172c55f3e7fec396050c330fc9f7b columnFamilyName testFamily 2024-11-12T07:52:51,767 DEBUG [StoreOpener-96b172c55f3e7fec396050c330fc9f7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:51,768 INFO [StoreOpener-96b172c55f3e7fec396050c330fc9f7b-1 {}] regionserver.HStore(327): Store=96b172c55f3e7fec396050c330fc9f7b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:52:51,768 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:52:51,769 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:52:51,769 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:52:51,770 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:52:51,770 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:52:51,773 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:52:51,776 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/96b172c55f3e7fec396050c330fc9f7b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:52:51,777 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 96b172c55f3e7fec396050c330fc9f7b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63930102, jitterRate=-0.047367244958877563}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T07:52:51,777 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:52:51,777 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 96b172c55f3e7fec396050c330fc9f7b: Running coprocessor pre-open hook at 1731397971763Writing region info on filesystem at 1731397971763Initializing all the Stores at 1731397971764 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397971764Cleaning up temporary data from old regions at 1731397971770 (+6 ms)Running coprocessor post-open hooks at 1731397971777 (+7 ms)Region opened successfully at 1731397971777 2024-11-12T07:52:51,778 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b., pid=27, masterSystemTime=1731397971749 2024-11-12T07:52:51,781 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. 2024-11-12T07:52:51,781 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. 2024-11-12T07:52:51,782 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=96b172c55f3e7fec396050c330fc9f7b, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:52:51,785 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96b172c55f3e7fec396050c330fc9f7b, server=f9fc96254ef8,43507,1731397896042 because future has completed 2024-11-12T07:52:51,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-12T07:52:51,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 96b172c55f3e7fec396050c330fc9f7b, server=f9fc96254ef8,43507,1731397896042 in 191 msec 2024-11-12T07:52:51,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-12T07:52:51,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=96b172c55f3e7fec396050c330fc9f7b, ASSIGN in 355 msec 2024-11-12T07:52:51,793 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:52:51,793 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397971793"}]},"ts":"1731397971793"} 2024-11-12T07:52:51,796 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-12T07:52:51,798 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:52:51,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2160 sec 2024-11-12T07:52:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T07:52:53,616 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-12T07:52:53,616 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-12T07:52:53,617 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:53,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-12T07:52:53,628 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:52:53,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-12T07:52:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2] 2024-11-12T07:52:53,643 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-12T07:52:53,643 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-12T07:52:53,647 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-12T07:52:53,649 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-12T07:52:55,612 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-12T07:52:55,612 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-12T07:52:55,615 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-12T07:52:55,615 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-12T07:52:57,292 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:52:57,321 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-12T07:52:59,648 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=f9fc96254ef8:43507 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6000ms, rpcTimeout=5999ms 2024-11-12T07:52:59,649 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6000ms, rpcTimeout=5999ms 2024-11-12T07:52:59,649 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 from cache 2024-11-12T07:52:59,665 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=549 (was 649), ProcessCount=11 (was 11), AvailableMemoryMB=6687 (was 6694) 2024-11-12T07:52:59,673 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=549, ProcessCount=11, AvailableMemoryMB=6687 2024-11-12T07:52:59,676 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42363 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:52:59,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42363 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-12T07:52:59,679 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:52:59,679 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:52:59,680 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42363 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-12T07:52:59,681 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:52:59,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T07:52:59,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741851_1027 (size=407) 2024-11-12T07:53:00,098 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 29b570581a5f6e7440c2a6a7401ba821, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a 2024-11-12T07:53:00,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741852_1028 (size=58) 2024-11-12T07:53:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-12T07:53:00,516 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:53:00,516 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 29b570581a5f6e7440c2a6a7401ba821, disabling compactions & flushes 2024-11-12T07:53:00,517 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. 2024-11-12T07:53:00,517 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. 2024-11-12T07:53:00,517 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. after waiting 0 ms 2024-11-12T07:53:00,517 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. 2024-11-12T07:53:00,517 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. 2024-11-12T07:53:00,518 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 29b570581a5f6e7440c2a6a7401ba821: Waiting for close lock at 1731397980516Disabling compacts and flushes for region at 1731397980516Disabling writes for close at 1731397980517 (+1 ms)Writing region close event to WAL at 1731397980517Closed at 1731397980517 2024-11-12T07:53:00,520 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:53:00,520 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1731397980520"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397980520"}]},"ts":"1731397980520"} 2024-11-12T07:53:00,524 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:53:00,526 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:53:00,526 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397980526"}]},"ts":"1731397980526"} 2024-11-12T07:53:00,528 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-12T07:53:00,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=29b570581a5f6e7440c2a6a7401ba821, ASSIGN}] 2024-11-12T07:53:00,530 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=29b570581a5f6e7440c2a6a7401ba821, ASSIGN 2024-11-12T07:53:00,531 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=29b570581a5f6e7440c2a6a7401ba821, ASSIGN; state=OFFLINE, location=f9fc96254ef8,43507,1731397896042; forceNewPlan=false, retain=false 2024-11-12T07:53:00,682 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=29b570581a5f6e7440c2a6a7401ba821, regionState=OPENING, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:53:00,686 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=29b570581a5f6e7440c2a6a7401ba821, ASSIGN because future has completed 2024-11-12T07:53:00,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29b570581a5f6e7440c2a6a7401ba821, server=f9fc96254ef8,43507,1731397896042}] 2024-11-12T07:53:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T07:53:00,855 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. 2024-11-12T07:53:00,855 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 29b570581a5f6e7440c2a6a7401ba821, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:53:00,856 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 29b570581a5f6e7440c2a6a7401ba821 2024-11-12T07:53:00,856 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:53:00,856 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 29b570581a5f6e7440c2a6a7401ba821 2024-11-12T07:53:00,856 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 29b570581a5f6e7440c2a6a7401ba821 2024-11-12T07:53:00,859 INFO [StoreOpener-29b570581a5f6e7440c2a6a7401ba821-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 29b570581a5f6e7440c2a6a7401ba821 2024-11-12T07:53:00,862 INFO [StoreOpener-29b570581a5f6e7440c2a6a7401ba821-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29b570581a5f6e7440c2a6a7401ba821 columnFamilyName testFamily 2024-11-12T07:53:00,862 DEBUG [StoreOpener-29b570581a5f6e7440c2a6a7401ba821-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:53:00,863 INFO [StoreOpener-29b570581a5f6e7440c2a6a7401ba821-1 {}] regionserver.HStore(327): Store=29b570581a5f6e7440c2a6a7401ba821/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:53:00,863 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 29b570581a5f6e7440c2a6a7401ba821 2024-11-12T07:53:00,865 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNext/29b570581a5f6e7440c2a6a7401ba821 2024-11-12T07:53:00,866 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNext/29b570581a5f6e7440c2a6a7401ba821 2024-11-12T07:53:00,866 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 29b570581a5f6e7440c2a6a7401ba821 2024-11-12T07:53:00,867 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 29b570581a5f6e7440c2a6a7401ba821 2024-11-12T07:53:00,869 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 29b570581a5f6e7440c2a6a7401ba821 2024-11-12T07:53:00,872 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNext/29b570581a5f6e7440c2a6a7401ba821/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:53:00,873 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 29b570581a5f6e7440c2a6a7401ba821; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61239495, jitterRate=-0.08746041357517242}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T07:53:00,873 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 29b570581a5f6e7440c2a6a7401ba821 2024-11-12T07:53:00,874 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 29b570581a5f6e7440c2a6a7401ba821: Running coprocessor pre-open hook at 1731397980857Writing region info on filesystem at 1731397980857Initializing all the Stores at 1731397980859 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397980859Cleaning up temporary data from old regions at 1731397980867 (+8 ms)Running coprocessor post-open hooks at 1731397980873 (+6 ms)Region opened successfully at 1731397980874 (+1 ms) 2024-11-12T07:53:00,875 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821., pid=30, masterSystemTime=1731397980841 2024-11-12T07:53:00,877 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. 2024-11-12T07:53:00,878 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. 2024-11-12T07:53:00,878 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=29b570581a5f6e7440c2a6a7401ba821, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:53:00,881 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29b570581a5f6e7440c2a6a7401ba821, server=f9fc96254ef8,43507,1731397896042 because future has completed 2024-11-12T07:53:00,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-12T07:53:00,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 29b570581a5f6e7440c2a6a7401ba821, server=f9fc96254ef8,43507,1731397896042 in 195 msec 2024-11-12T07:53:00,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-12T07:53:00,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=29b570581a5f6e7440c2a6a7401ba821, ASSIGN in 356 msec 2024-11-12T07:53:00,888 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:53:00,889 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397980888"}]},"ts":"1731397980888"} 2024-11-12T07:53:00,891 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-12T07:53:00,892 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:53:00,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2160 sec 2024-11-12T07:53:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-12T07:53:02,716 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-12T07:53:02,716 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-12T07:53:02,717 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:53:02,728 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-12T07:53:02,728 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:53:02,728 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-12T07:53:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2] 2024-11-12T07:53:02,740 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-12T07:53:02,740 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-12T07:53:02,743 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-12T07:53:02,745 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-12T07:53:02,746 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-12T07:53:02,747 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-12T07:53:03,323 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-12T07:53:04,287 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T07:53:05,612 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-12T07:53:05,612 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-12T07:53:06,362 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:53:08,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43507 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7368373797696569292' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-12T07:53:08,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43507 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:39384 deadline: 1731397994748, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7368373797696569292' does not exist 2024-11-12T07:53:08,752 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to f9fc96254ef8,43507,1731397896042 for closing scanner id = -7368373797696569292 for 29b570581a5f6e7440c2a6a7401ba821 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7368373797696569292' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:53:08,756 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=505 (was 549), ProcessCount=11 (was 11), AvailableMemoryMB=6683 (was 6687) 2024-11-12T07:53:08,764 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=505, ProcessCount=11, AvailableMemoryMB=6683 2024-11-12T07:53:08,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:53:08,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-12T07:53:08,768 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:53:08,769 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:53:08,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-12T07:53:08,770 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:53:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-12T07:53:08,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741853_1029 (size=423) 2024-11-12T07:53:09,186 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => da9627ad5149211c4e7296a7559f9bfe, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a 2024-11-12T07:53:09,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741854_1030 (size=74) 2024-11-12T07:53:09,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-12T07:53:09,600 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:53:09,600 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing da9627ad5149211c4e7296a7559f9bfe, disabling compactions & flushes 2024-11-12T07:53:09,601 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. 2024-11-12T07:53:09,601 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. 2024-11-12T07:53:09,601 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. after waiting 0 ms 2024-11-12T07:53:09,601 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. 2024-11-12T07:53:09,601 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. 2024-11-12T07:53:09,602 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for da9627ad5149211c4e7296a7559f9bfe: Waiting for close lock at 1731397989600Disabling compacts and flushes for region at 1731397989600Disabling writes for close at 1731397989601 (+1 ms)Writing region close event to WAL at 1731397989601Closed at 1731397989601 2024-11-12T07:53:09,605 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:53:09,605 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731397989605"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397989605"}]},"ts":"1731397989605"} 2024-11-12T07:53:09,609 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:53:09,611 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:53:09,611 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397989611"}]},"ts":"1731397989611"} 2024-11-12T07:53:09,614 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-12T07:53:09,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=da9627ad5149211c4e7296a7559f9bfe, ASSIGN}] 2024-11-12T07:53:09,616 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=da9627ad5149211c4e7296a7559f9bfe, ASSIGN 2024-11-12T07:53:09,617 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=da9627ad5149211c4e7296a7559f9bfe, ASSIGN; state=OFFLINE, location=f9fc96254ef8,43507,1731397896042; forceNewPlan=false, retain=false 2024-11-12T07:53:09,676 DEBUG [master/f9fc96254ef8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-12T07:53:09,768 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=da9627ad5149211c4e7296a7559f9bfe, regionState=OPENING, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:53:09,772 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=da9627ad5149211c4e7296a7559f9bfe, ASSIGN because future has completed 2024-11-12T07:53:09,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure da9627ad5149211c4e7296a7559f9bfe, server=f9fc96254ef8,43507,1731397896042}] 2024-11-12T07:53:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-12T07:53:09,936 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. 2024-11-12T07:53:09,937 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => da9627ad5149211c4e7296a7559f9bfe, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:53:09,937 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:09,937 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:53:09,937 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:09,937 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:09,939 INFO [StoreOpener-da9627ad5149211c4e7296a7559f9bfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:09,942 INFO [StoreOpener-da9627ad5149211c4e7296a7559f9bfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da9627ad5149211c4e7296a7559f9bfe columnFamilyName testFamily 2024-11-12T07:53:09,942 DEBUG [StoreOpener-da9627ad5149211c4e7296a7559f9bfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:53:09,942 INFO [StoreOpener-da9627ad5149211c4e7296a7559f9bfe-1 {}] regionserver.HStore(327): Store=da9627ad5149211c4e7296a7559f9bfe/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:53:09,943 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:09,944 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScannerAsync/da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:09,944 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScannerAsync/da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:09,945 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:09,945 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:09,947 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:09,951 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScannerAsync/da9627ad5149211c4e7296a7559f9bfe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:53:09,951 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened da9627ad5149211c4e7296a7559f9bfe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71241024, jitterRate=0.06157398223876953}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T07:53:09,951 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:09,952 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for da9627ad5149211c4e7296a7559f9bfe: Running coprocessor pre-open hook at 1731397989937Writing region info on filesystem at 1731397989937Initializing all the Stores at 1731397989939 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397989939Cleaning up temporary data from old regions at 1731397989945 (+6 ms)Running coprocessor post-open hooks at 1731397989952 (+7 ms)Region opened successfully at 1731397989952 2024-11-12T07:53:09,953 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., pid=33, masterSystemTime=1731397989926 2024-11-12T07:53:09,956 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. 2024-11-12T07:53:09,956 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. 2024-11-12T07:53:09,957 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=da9627ad5149211c4e7296a7559f9bfe, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,43507,1731397896042 2024-11-12T07:53:09,959 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure da9627ad5149211c4e7296a7559f9bfe, server=f9fc96254ef8,43507,1731397896042 because future has completed 2024-11-12T07:53:09,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-12T07:53:09,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure da9627ad5149211c4e7296a7559f9bfe, server=f9fc96254ef8,43507,1731397896042 in 189 msec 2024-11-12T07:53:09,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-12T07:53:09,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=da9627ad5149211c4e7296a7559f9bfe, ASSIGN in 349 msec 2024-11-12T07:53:09,966 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:53:09,967 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397989966"}]},"ts":"1731397989966"} 2024-11-12T07:53:09,969 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-12T07:53:09,970 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:53:09,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2040 sec 2024-11-12T07:53:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42363 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-12T07:53:11,807 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-12T07:53:11,807 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-12T07:53:11,808 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:53:11,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-12T07:53:11,819 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T07:53:11,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-12T07:53:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2] 2024-11-12T07:53:11,833 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-12T07:53:11,834 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-12T07:53:11,837 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-12T07:53:11,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-12T07:53:12,848 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=f9fc96254ef8:43507 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-11-12T07:53:12,848 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-11-12T07:53:12,848 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 from cache 2024-11-12T07:53:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-12T07:53:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2] 2024-11-12T07:53:13,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43507 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-12T07:53:14,848 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=f9fc96254ef8:43507 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=974ms, rpcTimeout=965ms 2024-11-12T07:53:14,848 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=974ms, rpcTimeout=965ms 2024-11-12T07:53:14,849 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., hostname=f9fc96254ef8,43507,1731397896042, seqNum=2 from cache 2024-11-12T07:53:14,864 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=427 (was 505), ProcessCount=11 (was 11), AvailableMemoryMB=6680 (was 6683) 2024-11-12T07:53:14,865 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:53:14,865 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:53:14,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:53:14,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:53:14,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:53:14,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T07:53:14,870 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:53:14,870 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:53:14,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:53:14,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:53:14,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:53:14,870 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T07:53:14,870 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2119115424, stopped=false 2024-11-12T07:53:14,871 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f9fc96254ef8,42363,1731397895269 2024-11-12T07:53:14,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T07:53:14,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T07:53:14,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:53:14,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:53:14,920 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T07:53:14,921 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:53:14,922 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:53:14,922 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:53:14,922 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:53:14,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:53:14,923 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f9fc96254ef8,43507,1731397896042' ***** 2024-11-12T07:53:14,923 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T07:53:14,923 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T07:53:14,924 INFO [RS:0;f9fc96254ef8:43507 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T07:53:14,924 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T07:53:14,924 INFO [RS:0;f9fc96254ef8:43507 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T07:53:14,925 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 02458ed77a017970a5c631399f75338a 2024-11-12T07:53:14,925 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:53:14,926 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 29b570581a5f6e7440c2a6a7401ba821 2024-11-12T07:53:14,926 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for c2da0bed673145aff0226f0ec2aa31e8 2024-11-12T07:53:14,926 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for d1b2734401ee4bc63a4d1307e3657981 2024-11-12T07:53:14,926 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 26d0ebdc343ac286c84ab8127870729b 2024-11-12T07:53:14,926 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 5be9542aae728d3ecaae7c77a6b7e6eb 2024-11-12T07:53:14,926 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 5e6fa6f9df49a0f74aee6be9e175a9d5 2024-11-12T07:53:14,926 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 02458ed77a017970a5c631399f75338a, disabling compactions & flushes 2024-11-12T07:53:14,926 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:14,926 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. 2024-11-12T07:53:14,926 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:53:14,926 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. 2024-11-12T07:53:14,927 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. after waiting 0 ms 2024-11-12T07:53:14,927 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(959): stopping server f9fc96254ef8,43507,1731397896042 2024-11-12T07:53:14,927 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T07:53:14,927 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. 2024-11-12T07:53:14,927 INFO [RS:0;f9fc96254ef8:43507 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f9fc96254ef8:43507. 2024-11-12T07:53:14,927 DEBUG [RS:0;f9fc96254ef8:43507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:53:14,927 DEBUG [RS:0;f9fc96254ef8:43507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:53:14,927 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T07:53:14,927 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T07:53:14,927 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T07:53:14,928 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T07:53:14,928 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-12T07:53:14,928 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T07:53:14,928 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T07:53:14,928 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T07:53:14,928 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T07:53:14,928 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 02458ed77a017970a5c631399f75338a, 1588230740, 26d0ebdc343ac286c84ab8127870729b, 29b570581a5f6e7440c2a6a7401ba821, 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, c2da0bed673145aff0226f0ec2aa31e8, d1b2734401ee4bc63a4d1307e3657981, da9627ad5149211c4e7296a7559f9bfe, de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:53:14,928 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T07:53:14,932 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-12T07:53:14,932 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 02458ed77a017970a5c631399f75338a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-12T07:53:14,950 INFO [regionserver/f9fc96254ef8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T07:53:14,950 INFO [regionserver/f9fc96254ef8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T07:53:14,984 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNextAsync/02458ed77a017970a5c631399f75338a/.tmp/testFamily/fb765a77c0af46f882d2f7151ce762bd is 61, key is row-0/testFamily:testQualifier/1731397945179/Put/seqid=0 2024-11-12T07:53:14,989 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/.tmp/info/131d50886b37435294a60c61ec8d4356 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8./info:regioninfo/1731397931095/Put/seqid=0 2024-11-12T07:53:14,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741856_1032 (size=5049) 2024-11-12T07:53:14,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741855_1031 (size=20564) 2024-11-12T07:53:14,995 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/.tmp/info/131d50886b37435294a60c61ec8d4356 2024-11-12T07:53:14,995 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNextAsync/02458ed77a017970a5c631399f75338a/.tmp/testFamily/fb765a77c0af46f882d2f7151ce762bd 2024-11-12T07:53:15,037 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNextAsync/02458ed77a017970a5c631399f75338a/.tmp/testFamily/fb765a77c0af46f882d2f7151ce762bd as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNextAsync/02458ed77a017970a5c631399f75338a/testFamily/fb765a77c0af46f882d2f7151ce762bd 2024-11-12T07:53:15,046 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNextAsync/02458ed77a017970a5c631399f75338a/testFamily/fb765a77c0af46f882d2f7151ce762bd, entries=4, sequenceid=8, filesize=4.9 K 2024-11-12T07:53:15,052 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/.tmp/ns/4d6d7cb4dd874ca687f49756ee33615e is 43, key is default/ns:d/1731397929619/Put/seqid=0 2024-11-12T07:53:15,055 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 02458ed77a017970a5c631399f75338a in 123ms, sequenceid=8, compaction requested=false 2024-11-12T07:53:15,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741857_1033 (size=5153) 2024-11-12T07:53:15,067 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNextAsync/02458ed77a017970a5c631399f75338a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-12T07:53:15,069 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. 2024-11-12T07:53:15,070 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 02458ed77a017970a5c631399f75338a: Waiting for close lock at 1731397994926Running coprocessor pre-close hooks at 1731397994926Disabling compacts and flushes for region at 1731397994926Disabling writes for close at 1731397994927 (+1 ms)Obtaining lock to block concurrent updates at 1731397994932 (+5 ms)Preparing flush snapshotting stores in 02458ed77a017970a5c631399f75338a at 1731397994932Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731397994940 (+8 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. at 1731397994941 (+1 ms)Flushing 02458ed77a017970a5c631399f75338a/testFamily: creating writer at 1731397994943 (+2 ms)Flushing 02458ed77a017970a5c631399f75338a/testFamily: appending metadata at 1731397994979 (+36 ms)Flushing 02458ed77a017970a5c631399f75338a/testFamily: closing flushed file at 1731397994982 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a180103: reopening flushed file at 1731397995035 (+53 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 02458ed77a017970a5c631399f75338a in 123ms, sequenceid=8, compaction requested=false at 1731397995055 (+20 ms)Writing region close event to WAL at 1731397995058 (+3 ms)Running coprocessor post-close hooks at 1731397995068 (+10 ms)Closed at 1731397995069 (+1 ms) 2024-11-12T07:53:15,070 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731397942125.02458ed77a017970a5c631399f75338a. 2024-11-12T07:53:15,070 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing de9251dafa4bdb68ff37f1d9fec5985a, disabling compactions & flushes 2024-11-12T07:53:15,070 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. 2024-11-12T07:53:15,070 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. 2024-11-12T07:53:15,071 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. after waiting 0 ms 2024-11-12T07:53:15,071 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. 2024-11-12T07:53:15,071 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing de9251dafa4bdb68ff37f1d9fec5985a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-12T07:53:15,087 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScanner/de9251dafa4bdb68ff37f1d9fec5985a/.tmp/testFamily/642911f88d9f43dba56c2ce3cdfbb352 is 61, key is row-0/testFamily:testQualifier/1731397957363/Put/seqid=0 2024-11-12T07:53:15,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741858_1034 (size=5049) 2024-11-12T07:53:15,129 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 26d0ebdc343ac286c84ab8127870729b, 29b570581a5f6e7440c2a6a7401ba821, 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, c2da0bed673145aff0226f0ec2aa31e8, d1b2734401ee4bc63a4d1307e3657981, da9627ad5149211c4e7296a7559f9bfe, de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:53:15,329 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 26d0ebdc343ac286c84ab8127870729b, 29b570581a5f6e7440c2a6a7401ba821, 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, c2da0bed673145aff0226f0ec2aa31e8, d1b2734401ee4bc63a4d1307e3657981, da9627ad5149211c4e7296a7559f9bfe, de9251dafa4bdb68ff37f1d9fec5985a 2024-11-12T07:53:15,376 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-12T07:53:15,439 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:53:15,460 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/.tmp/ns/4d6d7cb4dd874ca687f49756ee33615e 2024-11-12T07:53:15,477 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-12T07:53:15,488 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/.tmp/table/c9e9b81ae515409eb2918716b10f105c is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1731397931113/Put/seqid=0 2024-11-12T07:53:15,493 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScanner/de9251dafa4bdb68ff37f1d9fec5985a/.tmp/testFamily/642911f88d9f43dba56c2ce3cdfbb352 2024-11-12T07:53:15,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741859_1035 (size=6699) 2024-11-12T07:53:15,501 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScanner/de9251dafa4bdb68ff37f1d9fec5985a/.tmp/testFamily/642911f88d9f43dba56c2ce3cdfbb352 as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScanner/de9251dafa4bdb68ff37f1d9fec5985a/testFamily/642911f88d9f43dba56c2ce3cdfbb352 2024-11-12T07:53:15,509 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScanner/de9251dafa4bdb68ff37f1d9fec5985a/testFamily/642911f88d9f43dba56c2ce3cdfbb352, entries=4, sequenceid=8, filesize=4.9 K 2024-11-12T07:53:15,510 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for de9251dafa4bdb68ff37f1d9fec5985a in 439ms, sequenceid=8, compaction requested=false 2024-11-12T07:53:15,515 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScanner/de9251dafa4bdb68ff37f1d9fec5985a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-12T07:53:15,516 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. 2024-11-12T07:53:15,516 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for de9251dafa4bdb68ff37f1d9fec5985a: Waiting for close lock at 1731397995070Running coprocessor pre-close hooks at 1731397995070Disabling compacts and flushes for region at 1731397995070Disabling writes for close at 1731397995071 (+1 ms)Obtaining lock to block concurrent updates at 1731397995071Preparing flush snapshotting stores in de9251dafa4bdb68ff37f1d9fec5985a at 1731397995071Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731397995071Flushing stores of testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. at 1731397995072 (+1 ms)Flushing de9251dafa4bdb68ff37f1d9fec5985a/testFamily: creating writer at 1731397995072Flushing de9251dafa4bdb68ff37f1d9fec5985a/testFamily: appending metadata at 1731397995086 (+14 ms)Flushing de9251dafa4bdb68ff37f1d9fec5985a/testFamily: closing flushed file at 1731397995087 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9b044d8: reopening flushed file at 1731397995500 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for de9251dafa4bdb68ff37f1d9fec5985a in 439ms, sequenceid=8, compaction requested=false at 1731397995511 (+11 ms)Writing region close event to WAL at 1731397995511Running coprocessor post-close hooks at 1731397995516 (+5 ms)Closed at 1731397995516 2024-11-12T07:53:15,517 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1731397954321.de9251dafa4bdb68ff37f1d9fec5985a. 2024-11-12T07:53:15,517 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 29b570581a5f6e7440c2a6a7401ba821, disabling compactions & flushes 2024-11-12T07:53:15,517 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. 2024-11-12T07:53:15,517 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. 2024-11-12T07:53:15,517 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. after waiting 0 ms 2024-11-12T07:53:15,517 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. 2024-11-12T07:53:15,517 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 29b570581a5f6e7440c2a6a7401ba821 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-12T07:53:15,529 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 26d0ebdc343ac286c84ab8127870729b, 29b570581a5f6e7440c2a6a7401ba821, 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, c2da0bed673145aff0226f0ec2aa31e8, d1b2734401ee4bc63a4d1307e3657981, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:15,537 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNext/29b570581a5f6e7440c2a6a7401ba821/.tmp/testFamily/b8441e3809a04e838930417704fe8879 is 61, key is row-0/testFamily:testQualifier/1731397982732/Put/seqid=0 2024-11-12T07:53:15,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741860_1036 (size=5049) 2024-11-12T07:53:15,612 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-12T07:53:15,612 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-12T07:53:15,613 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-12T07:53:15,613 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-12T07:53:15,730 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 26d0ebdc343ac286c84ab8127870729b, 29b570581a5f6e7440c2a6a7401ba821, 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, c2da0bed673145aff0226f0ec2aa31e8, d1b2734401ee4bc63a4d1307e3657981, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:15,897 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/.tmp/table/c9e9b81ae515409eb2918716b10f105c 2024-11-12T07:53:15,913 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/.tmp/info/131d50886b37435294a60c61ec8d4356 as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/info/131d50886b37435294a60c61ec8d4356 2024-11-12T07:53:15,923 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/info/131d50886b37435294a60c61ec8d4356, entries=100, sequenceid=56, filesize=20.1 K 2024-11-12T07:53:15,925 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/.tmp/ns/4d6d7cb4dd874ca687f49756ee33615e as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/ns/4d6d7cb4dd874ca687f49756ee33615e 2024-11-12T07:53:15,930 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-12T07:53:15,930 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1325): Online Regions={29b570581a5f6e7440c2a6a7401ba821=hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821., 1588230740=hbase:meta,,1.1588230740, c2da0bed673145aff0226f0ec2aa31e8=testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8., d1b2734401ee4bc63a4d1307e3657981=testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981., 26d0ebdc343ac286c84ab8127870729b=testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b., 5be9542aae728d3ecaae7c77a6b7e6eb=hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb., 5e6fa6f9df49a0f74aee6be9e175a9d5=testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5., da9627ad5149211c4e7296a7559f9bfe=testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., 96b172c55f3e7fec396050c330fc9f7b=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b.} 2024-11-12T07:53:15,930 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 26d0ebdc343ac286c84ab8127870729b, 29b570581a5f6e7440c2a6a7401ba821, 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, c2da0bed673145aff0226f0ec2aa31e8, d1b2734401ee4bc63a4d1307e3657981, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:15,933 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/ns/4d6d7cb4dd874ca687f49756ee33615e, entries=2, sequenceid=56, filesize=5.0 K 2024-11-12T07:53:15,935 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/.tmp/table/c9e9b81ae515409eb2918716b10f105c as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/table/c9e9b81ae515409eb2918716b10f105c 2024-11-12T07:53:15,943 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNext/29b570581a5f6e7440c2a6a7401ba821/.tmp/testFamily/b8441e3809a04e838930417704fe8879 2024-11-12T07:53:15,943 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/table/c9e9b81ae515409eb2918716b10f105c, entries=20, sequenceid=56, filesize=6.5 K 2024-11-12T07:53:15,945 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1015ms, sequenceid=56, compaction requested=false 2024-11-12T07:53:15,951 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-12T07:53:15,952 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T07:53:15,952 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T07:53:15,952 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731397994928Running coprocessor pre-close hooks at 1731397994928Disabling compacts and flushes for region at 1731397994928Disabling writes for close at 1731397994928Obtaining lock to block concurrent updates at 1731397994932 (+4 ms)Preparing flush snapshotting stores in 1588230740 at 1731397994932Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1731397994939 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731397994941 (+2 ms)Flushing 1588230740/info: creating writer at 1731397994943 (+2 ms)Flushing 1588230740/info: appending metadata at 1731397994984 (+41 ms)Flushing 1588230740/info: closing flushed file at 1731397994984Flushing 1588230740/ns: creating writer at 1731397995035 (+51 ms)Flushing 1588230740/ns: appending metadata at 1731397995051 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731397995051Flushing 1588230740/table: creating writer at 1731397995469 (+418 ms)Flushing 1588230740/table: appending metadata at 1731397995487 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731397995488 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19f3c7b: reopening flushed file at 1731397995911 (+423 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@545fd025: reopening flushed file at 1731397995923 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f03df3c: reopening flushed file at 1731397995934 (+11 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1015ms, sequenceid=56, compaction requested=false at 1731397995945 (+11 ms)Writing region close event to WAL at 1731397995946 (+1 ms)Running coprocessor post-close hooks at 1731397995951 (+5 ms)Closed at 1731397995952 (+1 ms) 2024-11-12T07:53:15,952 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T07:53:15,953 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNext/29b570581a5f6e7440c2a6a7401ba821/.tmp/testFamily/b8441e3809a04e838930417704fe8879 as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNext/29b570581a5f6e7440c2a6a7401ba821/testFamily/b8441e3809a04e838930417704fe8879 2024-11-12T07:53:15,961 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNext/29b570581a5f6e7440c2a6a7401ba821/testFamily/b8441e3809a04e838930417704fe8879, entries=4, sequenceid=8, filesize=4.9 K 2024-11-12T07:53:15,962 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 29b570581a5f6e7440c2a6a7401ba821 in 445ms, sequenceid=8, compaction requested=false 2024-11-12T07:53:15,968 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnNext/29b570581a5f6e7440c2a6a7401ba821/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-12T07:53:15,968 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. 2024-11-12T07:53:15,969 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 29b570581a5f6e7440c2a6a7401ba821: Waiting for close lock at 1731397995517Running coprocessor pre-close hooks at 1731397995517Disabling compacts and flushes for region at 1731397995517Disabling writes for close at 1731397995517Obtaining lock to block concurrent updates at 1731397995517Preparing flush snapshotting stores in 29b570581a5f6e7440c2a6a7401ba821 at 1731397995517Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731397995517Flushing stores of hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. at 1731397995518 (+1 ms)Flushing 29b570581a5f6e7440c2a6a7401ba821/testFamily: creating writer at 1731397995518Flushing 29b570581a5f6e7440c2a6a7401ba821/testFamily: appending metadata at 1731397995536 (+18 ms)Flushing 29b570581a5f6e7440c2a6a7401ba821/testFamily: closing flushed file at 1731397995536Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d69e4b5: reopening flushed file at 1731397995952 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 29b570581a5f6e7440c2a6a7401ba821 in 445ms, sequenceid=8, compaction requested=false at 1731397995962 (+10 ms)Writing region close event to WAL at 1731397995963 (+1 ms)Running coprocessor post-close hooks at 1731397995968 (+5 ms)Closed at 1731397995968 2024-11-12T07:53:15,969 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1731397979676.29b570581a5f6e7440c2a6a7401ba821. 2024-11-12T07:53:15,969 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c2da0bed673145aff0226f0ec2aa31e8, disabling compactions & flushes 2024-11-12T07:53:15,969 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. 2024-11-12T07:53:15,969 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. 2024-11-12T07:53:15,969 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. after waiting 0 ms 2024-11-12T07:53:15,969 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. 2024-11-12T07:53:15,969 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c2da0bed673145aff0226f0ec2aa31e8 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-12T07:53:15,987 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c2da0bed673145aff0226f0ec2aa31e8/.tmp/testFamily/6b517e318fba447f8356d255a558c482 is 61, key is row-0/testFamily:testQualifier/1731397932923/Put/seqid=0 2024-11-12T07:53:15,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741861_1037 (size=5049) 2024-11-12T07:53:16,130 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 26d0ebdc343ac286c84ab8127870729b, 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, c2da0bed673145aff0226f0ec2aa31e8, d1b2734401ee4bc63a4d1307e3657981, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:16,331 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 26d0ebdc343ac286c84ab8127870729b, 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, c2da0bed673145aff0226f0ec2aa31e8, d1b2734401ee4bc63a4d1307e3657981, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:16,395 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c2da0bed673145aff0226f0ec2aa31e8/.tmp/testFamily/6b517e318fba447f8356d255a558c482 2024-11-12T07:53:16,408 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c2da0bed673145aff0226f0ec2aa31e8/.tmp/testFamily/6b517e318fba447f8356d255a558c482 as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c2da0bed673145aff0226f0ec2aa31e8/testFamily/6b517e318fba447f8356d255a558c482 2024-11-12T07:53:16,415 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c2da0bed673145aff0226f0ec2aa31e8/testFamily/6b517e318fba447f8356d255a558c482, entries=4, sequenceid=8, filesize=4.9 K 2024-11-12T07:53:16,417 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c2da0bed673145aff0226f0ec2aa31e8 in 447ms, sequenceid=8, compaction requested=false 2024-11-12T07:53:16,421 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c2da0bed673145aff0226f0ec2aa31e8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-12T07:53:16,422 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. 2024-11-12T07:53:16,422 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c2da0bed673145aff0226f0ec2aa31e8: Waiting for close lock at 1731397995969Running coprocessor pre-close hooks at 1731397995969Disabling compacts and flushes for region at 1731397995969Disabling writes for close at 1731397995969Obtaining lock to block concurrent updates at 1731397995969Preparing flush snapshotting stores in c2da0bed673145aff0226f0ec2aa31e8 at 1731397995969Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731397995970 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. at 1731397995970Flushing c2da0bed673145aff0226f0ec2aa31e8/testFamily: creating writer at 1731397995970Flushing c2da0bed673145aff0226f0ec2aa31e8/testFamily: appending metadata at 1731397995987 (+17 ms)Flushing c2da0bed673145aff0226f0ec2aa31e8/testFamily: closing flushed file at 1731397995987Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@571075a6: reopening flushed file at 1731397996406 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c2da0bed673145aff0226f0ec2aa31e8 in 447ms, sequenceid=8, compaction requested=false at 1731397996417 (+11 ms)Writing region close event to WAL at 1731397996417Running coprocessor post-close hooks at 1731397996422 (+5 ms)Closed at 1731397996422 2024-11-12T07:53:16,423 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731397929833.c2da0bed673145aff0226f0ec2aa31e8. 2024-11-12T07:53:16,423 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d1b2734401ee4bc63a4d1307e3657981, disabling compactions & flushes 2024-11-12T07:53:16,423 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. 2024-11-12T07:53:16,423 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. 2024-11-12T07:53:16,423 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. after waiting 0 ms 2024-11-12T07:53:16,423 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. 2024-11-12T07:53:16,423 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d1b2734401ee4bc63a4d1307e3657981 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-12T07:53:16,440 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNextAsync/d1b2734401ee4bc63a4d1307e3657981/.tmp/testFamily/787e03d6115a40a5b1202e57833531a0 is 61, key is row-0/testFamily:testQualifier/1731397961438/Put/seqid=0 2024-11-12T07:53:16,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741862_1038 (size=5049) 2024-11-12T07:53:16,531 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 26d0ebdc343ac286c84ab8127870729b, 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, d1b2734401ee4bc63a4d1307e3657981, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:16,654 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=f9fc96254ef8:42363 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 10 more 2024-11-12T07:53:16,732 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 26d0ebdc343ac286c84ab8127870729b, 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, d1b2734401ee4bc63a4d1307e3657981, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:16,847 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNextAsync/d1b2734401ee4bc63a4d1307e3657981/.tmp/testFamily/787e03d6115a40a5b1202e57833531a0 2024-11-12T07:53:16,862 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNextAsync/d1b2734401ee4bc63a4d1307e3657981/.tmp/testFamily/787e03d6115a40a5b1202e57833531a0 as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNextAsync/d1b2734401ee4bc63a4d1307e3657981/testFamily/787e03d6115a40a5b1202e57833531a0 2024-11-12T07:53:16,871 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNextAsync/d1b2734401ee4bc63a4d1307e3657981/testFamily/787e03d6115a40a5b1202e57833531a0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-12T07:53:16,872 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d1b2734401ee4bc63a4d1307e3657981 in 449ms, sequenceid=8, compaction requested=false 2024-11-12T07:53:16,877 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNextAsync/d1b2734401ee4bc63a4d1307e3657981/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-12T07:53:16,878 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. 2024-11-12T07:53:16,878 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d1b2734401ee4bc63a4d1307e3657981: Waiting for close lock at 1731397996423Running coprocessor pre-close hooks at 1731397996423Disabling compacts and flushes for region at 1731397996423Disabling writes for close at 1731397996423Obtaining lock to block concurrent updates at 1731397996423Preparing flush snapshotting stores in d1b2734401ee4bc63a4d1307e3657981 at 1731397996423Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731397996423Flushing stores of testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. at 1731397996424 (+1 ms)Flushing d1b2734401ee4bc63a4d1307e3657981/testFamily: creating writer at 1731397996424Flushing d1b2734401ee4bc63a4d1307e3657981/testFamily: appending metadata at 1731397996440 (+16 ms)Flushing d1b2734401ee4bc63a4d1307e3657981/testFamily: closing flushed file at 1731397996440Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f362064: reopening flushed file at 1731397996861 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d1b2734401ee4bc63a4d1307e3657981 in 449ms, sequenceid=8, compaction requested=false at 1731397996872 (+11 ms)Writing region close event to WAL at 1731397996873 (+1 ms)Running coprocessor post-close hooks at 1731397996878 (+5 ms)Closed at 1731397996878 2024-11-12T07:53:16,878 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1731397960401.d1b2734401ee4bc63a4d1307e3657981. 2024-11-12T07:53:16,878 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 26d0ebdc343ac286c84ab8127870729b, disabling compactions & flushes 2024-11-12T07:53:16,878 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. 2024-11-12T07:53:16,878 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. 2024-11-12T07:53:16,879 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. after waiting 0 ms 2024-11-12T07:53:16,879 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. 2024-11-12T07:53:16,879 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 26d0ebdc343ac286c84ab8127870729b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-12T07:53:16,900 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNext/26d0ebdc343ac286c84ab8127870729b/.tmp/testFamily/24066d2f1af745ebaf891aeadf34f481 is 61, key is row-0/testFamily:testQualifier/1731397967534/Put/seqid=0 2024-11-12T07:53:16,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741863_1039 (size=5049) 2024-11-12T07:53:16,932 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-11-12T07:53:16,932 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1325): Online Regions={26d0ebdc343ac286c84ab8127870729b=testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b., 5be9542aae728d3ecaae7c77a6b7e6eb=hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb., 5e6fa6f9df49a0f74aee6be9e175a9d5=testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5., da9627ad5149211c4e7296a7559f9bfe=testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., 96b172c55f3e7fec396050c330fc9f7b=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b.} 2024-11-12T07:53:16,932 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 26d0ebdc343ac286c84ab8127870729b, 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:17,132 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 26d0ebdc343ac286c84ab8127870729b, 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:17,307 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNext/26d0ebdc343ac286c84ab8127870729b/.tmp/testFamily/24066d2f1af745ebaf891aeadf34f481 2024-11-12T07:53:17,319 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNext/26d0ebdc343ac286c84ab8127870729b/.tmp/testFamily/24066d2f1af745ebaf891aeadf34f481 as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNext/26d0ebdc343ac286c84ab8127870729b/testFamily/24066d2f1af745ebaf891aeadf34f481 2024-11-12T07:53:17,327 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNext/26d0ebdc343ac286c84ab8127870729b/testFamily/24066d2f1af745ebaf891aeadf34f481, entries=4, sequenceid=8, filesize=4.9 K 2024-11-12T07:53:17,329 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 26d0ebdc343ac286c84ab8127870729b in 450ms, sequenceid=8, compaction requested=false 2024-11-12T07:53:17,333 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 26d0ebdc343ac286c84ab8127870729b, 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:17,333 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnNext/26d0ebdc343ac286c84ab8127870729b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-12T07:53:17,334 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. 2024-11-12T07:53:17,334 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 26d0ebdc343ac286c84ab8127870729b: Waiting for close lock at 1731397996878Running coprocessor pre-close hooks at 1731397996878Disabling compacts and flushes for region at 1731397996878Disabling writes for close at 1731397996879 (+1 ms)Obtaining lock to block concurrent updates at 1731397996879Preparing flush snapshotting stores in 26d0ebdc343ac286c84ab8127870729b at 1731397996879Finished memstore snapshotting testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731397996879Flushing stores of testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. at 1731397996880 (+1 ms)Flushing 26d0ebdc343ac286c84ab8127870729b/testFamily: creating writer at 1731397996880Flushing 26d0ebdc343ac286c84ab8127870729b/testFamily: appending metadata at 1731397996900 (+20 ms)Flushing 26d0ebdc343ac286c84ab8127870729b/testFamily: closing flushed file at 1731397996900Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16c44ec6: reopening flushed file at 1731397997318 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 26d0ebdc343ac286c84ab8127870729b in 450ms, sequenceid=8, compaction requested=false at 1731397997329 (+11 ms)Writing region close event to WAL at 1731397997329Running coprocessor post-close hooks at 1731397997334 (+5 ms)Closed at 1731397997334 2024-11-12T07:53:17,334 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1731397964486.26d0ebdc343ac286c84ab8127870729b. 2024-11-12T07:53:17,335 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5be9542aae728d3ecaae7c77a6b7e6eb, disabling compactions & flushes 2024-11-12T07:53:17,335 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. 2024-11-12T07:53:17,335 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. 2024-11-12T07:53:17,335 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. after waiting 0 ms 2024-11-12T07:53:17,335 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. 2024-11-12T07:53:17,335 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5be9542aae728d3ecaae7c77a6b7e6eb 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-12T07:53:17,350 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScanner/5be9542aae728d3ecaae7c77a6b7e6eb/.tmp/testFamily/4735fead63b74b5fbac742cfdb08d605 is 61, key is row-0/testFamily:testQualifier/1731397936079/Put/seqid=0 2024-11-12T07:53:17,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741864_1040 (size=5049) 2024-11-12T07:53:17,533 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:17,734 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 5be9542aae728d3ecaae7c77a6b7e6eb, 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:17,756 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScanner/5be9542aae728d3ecaae7c77a6b7e6eb/.tmp/testFamily/4735fead63b74b5fbac742cfdb08d605 2024-11-12T07:53:17,770 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScanner/5be9542aae728d3ecaae7c77a6b7e6eb/.tmp/testFamily/4735fead63b74b5fbac742cfdb08d605 as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScanner/5be9542aae728d3ecaae7c77a6b7e6eb/testFamily/4735fead63b74b5fbac742cfdb08d605 2024-11-12T07:53:17,776 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScanner/5be9542aae728d3ecaae7c77a6b7e6eb/testFamily/4735fead63b74b5fbac742cfdb08d605, entries=4, sequenceid=8, filesize=4.9 K 2024-11-12T07:53:17,777 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5be9542aae728d3ecaae7c77a6b7e6eb in 442ms, sequenceid=8, compaction requested=false 2024-11-12T07:53:17,782 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScanner/5be9542aae728d3ecaae7c77a6b7e6eb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-12T07:53:17,782 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. 2024-11-12T07:53:17,782 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5be9542aae728d3ecaae7c77a6b7e6eb: Waiting for close lock at 1731397997335Running coprocessor pre-close hooks at 1731397997335Disabling compacts and flushes for region at 1731397997335Disabling writes for close at 1731397997335Obtaining lock to block concurrent updates at 1731397997335Preparing flush snapshotting stores in 5be9542aae728d3ecaae7c77a6b7e6eb at 1731397997335Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731397997335Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. at 1731397997336 (+1 ms)Flushing 5be9542aae728d3ecaae7c77a6b7e6eb/testFamily: creating writer at 1731397997336Flushing 5be9542aae728d3ecaae7c77a6b7e6eb/testFamily: appending metadata at 1731397997350 (+14 ms)Flushing 5be9542aae728d3ecaae7c77a6b7e6eb/testFamily: closing flushed file at 1731397997350Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45692c74: reopening flushed file at 1731397997768 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5be9542aae728d3ecaae7c77a6b7e6eb in 442ms, sequenceid=8, compaction requested=false at 1731397997777 (+9 ms)Writing region close event to WAL at 1731397997778 (+1 ms)Running coprocessor post-close hooks at 1731397997782 (+4 ms)Closed at 1731397997782 2024-11-12T07:53:17,783 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731397933014.5be9542aae728d3ecaae7c77a6b7e6eb. 2024-11-12T07:53:17,783 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5e6fa6f9df49a0f74aee6be9e175a9d5, disabling compactions & flushes 2024-11-12T07:53:17,783 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. 2024-11-12T07:53:17,783 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. 2024-11-12T07:53:17,783 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. after waiting 0 ms 2024-11-12T07:53:17,783 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. 2024-11-12T07:53:17,783 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5e6fa6f9df49a0f74aee6be9e175a9d5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-12T07:53:17,799 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextException/5e6fa6f9df49a0f74aee6be9e175a9d5/.tmp/testFamily/7ec50bc5d1c544aebc2f1003316a3728 is 61, key is row-0/testFamily:testQualifier/1731397954265/Put/seqid=0 2024-11-12T07:53:17,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741865_1041 (size=5049) 2024-11-12T07:53:17,934 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-12T07:53:17,934 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1325): Online Regions={5e6fa6f9df49a0f74aee6be9e175a9d5=testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5., da9627ad5149211c4e7296a7559f9bfe=testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., 96b172c55f3e7fec396050c330fc9f7b=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b.} 2024-11-12T07:53:17,935 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:18,135 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 5e6fa6f9df49a0f74aee6be9e175a9d5, 96b172c55f3e7fec396050c330fc9f7b, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:18,205 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextException/5e6fa6f9df49a0f74aee6be9e175a9d5/.tmp/testFamily/7ec50bc5d1c544aebc2f1003316a3728 2024-11-12T07:53:18,219 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextException/5e6fa6f9df49a0f74aee6be9e175a9d5/.tmp/testFamily/7ec50bc5d1c544aebc2f1003316a3728 as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextException/5e6fa6f9df49a0f74aee6be9e175a9d5/testFamily/7ec50bc5d1c544aebc2f1003316a3728 2024-11-12T07:53:18,228 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextException/5e6fa6f9df49a0f74aee6be9e175a9d5/testFamily/7ec50bc5d1c544aebc2f1003316a3728, entries=4, sequenceid=8, filesize=4.9 K 2024-11-12T07:53:18,229 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5e6fa6f9df49a0f74aee6be9e175a9d5 in 446ms, sequenceid=8, compaction requested=false 2024-11-12T07:53:18,234 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testRetryOutOfOrderScannerNextException/5e6fa6f9df49a0f74aee6be9e175a9d5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-12T07:53:18,234 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. 2024-11-12T07:53:18,234 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5e6fa6f9df49a0f74aee6be9e175a9d5: Waiting for close lock at 1731397997783Running coprocessor pre-close hooks at 1731397997783Disabling compacts and flushes for region at 1731397997783Disabling writes for close at 1731397997783Obtaining lock to block concurrent updates at 1731397997783Preparing flush snapshotting stores in 5e6fa6f9df49a0f74aee6be9e175a9d5 at 1731397997783Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731397997783Flushing stores of testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. at 1731397997784 (+1 ms)Flushing 5e6fa6f9df49a0f74aee6be9e175a9d5/testFamily: creating writer at 1731397997784Flushing 5e6fa6f9df49a0f74aee6be9e175a9d5/testFamily: appending metadata at 1731397997799 (+15 ms)Flushing 5e6fa6f9df49a0f74aee6be9e175a9d5/testFamily: closing flushed file at 1731397997799Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29976efe: reopening flushed file at 1731397998218 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5e6fa6f9df49a0f74aee6be9e175a9d5 in 446ms, sequenceid=8, compaction requested=false at 1731397998229 (+11 ms)Writing region close event to WAL at 1731397998230 (+1 ms)Running coprocessor post-close hooks at 1731397998234 (+4 ms)Closed at 1731397998234 2024-11-12T07:53:18,235 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1731397951227.5e6fa6f9df49a0f74aee6be9e175a9d5. 2024-11-12T07:53:18,235 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing da9627ad5149211c4e7296a7559f9bfe, disabling compactions & flushes 2024-11-12T07:53:18,235 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. 2024-11-12T07:53:18,235 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. 2024-11-12T07:53:18,235 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. after waiting 0 ms 2024-11-12T07:53:18,235 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. 2024-11-12T07:53:18,235 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing da9627ad5149211c4e7296a7559f9bfe 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-12T07:53:18,252 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScannerAsync/da9627ad5149211c4e7296a7559f9bfe/.tmp/testFamily/2c9a51fbdafc427ba4185524462e41c3 is 61, key is row-0/testFamily:testQualifier/1731397991824/Put/seqid=0 2024-11-12T07:53:18,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741866_1042 (size=5049) 2024-11-12T07:53:18,335 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 96b172c55f3e7fec396050c330fc9f7b, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:18,536 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 96b172c55f3e7fec396050c330fc9f7b, da9627ad5149211c4e7296a7559f9bfe 2024-11-12T07:53:18,659 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScannerAsync/da9627ad5149211c4e7296a7559f9bfe/.tmp/testFamily/2c9a51fbdafc427ba4185524462e41c3 2024-11-12T07:53:18,675 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScannerAsync/da9627ad5149211c4e7296a7559f9bfe/.tmp/testFamily/2c9a51fbdafc427ba4185524462e41c3 as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScannerAsync/da9627ad5149211c4e7296a7559f9bfe/testFamily/2c9a51fbdafc427ba4185524462e41c3 2024-11-12T07:53:18,685 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScannerAsync/da9627ad5149211c4e7296a7559f9bfe/testFamily/2c9a51fbdafc427ba4185524462e41c3, entries=4, sequenceid=8, filesize=4.9 K 2024-11-12T07:53:18,686 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for da9627ad5149211c4e7296a7559f9bfe in 451ms, sequenceid=8, compaction requested=false 2024-11-12T07:53:18,691 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/default/testNormalScanTimeoutOnOpenScannerAsync/da9627ad5149211c4e7296a7559f9bfe/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-12T07:53:18,692 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. 2024-11-12T07:53:18,692 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for da9627ad5149211c4e7296a7559f9bfe: Waiting for close lock at 1731397998235Running coprocessor pre-close hooks at 1731397998235Disabling compacts and flushes for region at 1731397998235Disabling writes for close at 1731397998235Obtaining lock to block concurrent updates at 1731397998235Preparing flush snapshotting stores in da9627ad5149211c4e7296a7559f9bfe at 1731397998235Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731397998235Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. at 1731397998236 (+1 ms)Flushing da9627ad5149211c4e7296a7559f9bfe/testFamily: creating writer at 1731397998236Flushing da9627ad5149211c4e7296a7559f9bfe/testFamily: appending metadata at 1731397998252 (+16 ms)Flushing da9627ad5149211c4e7296a7559f9bfe/testFamily: closing flushed file at 1731397998252Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39d807b3: reopening flushed file at 1731397998674 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for da9627ad5149211c4e7296a7559f9bfe in 451ms, sequenceid=8, compaction requested=false at 1731397998686 (+12 ms)Writing region close event to WAL at 1731397998687 (+1 ms)Running coprocessor post-close hooks at 1731397998692 (+5 ms)Closed at 1731397998692 2024-11-12T07:53:18,692 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731397988765.da9627ad5149211c4e7296a7559f9bfe. 2024-11-12T07:53:18,692 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 96b172c55f3e7fec396050c330fc9f7b, disabling compactions & flushes 2024-11-12T07:53:18,692 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. 2024-11-12T07:53:18,692 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. 2024-11-12T07:53:18,692 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. after waiting 0 ms 2024-11-12T07:53:18,692 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. 2024-11-12T07:53:18,692 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 96b172c55f3e7fec396050c330fc9f7b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-12T07:53:18,707 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/96b172c55f3e7fec396050c330fc9f7b/.tmp/testFamily/a344f7aa279c4f349f71b5999ed58dc0 is 61, key is row-0/testFamily:testQualifier/1731397973632/Put/seqid=0 2024-11-12T07:53:18,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741867_1043 (size=5049) 2024-11-12T07:53:18,736 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:53:18,937 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T07:53:18,937 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1325): Online Regions={96b172c55f3e7fec396050c330fc9f7b=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b.} 2024-11-12T07:53:18,937 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:53:19,113 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/96b172c55f3e7fec396050c330fc9f7b/.tmp/testFamily/a344f7aa279c4f349f71b5999ed58dc0 2024-11-12T07:53:19,128 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/96b172c55f3e7fec396050c330fc9f7b/.tmp/testFamily/a344f7aa279c4f349f71b5999ed58dc0 as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/96b172c55f3e7fec396050c330fc9f7b/testFamily/a344f7aa279c4f349f71b5999ed58dc0 2024-11-12T07:53:19,135 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/96b172c55f3e7fec396050c330fc9f7b/testFamily/a344f7aa279c4f349f71b5999ed58dc0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-12T07:53:19,137 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 96b172c55f3e7fec396050c330fc9f7b in 445ms, sequenceid=8, compaction requested=false 2024-11-12T07:53:19,137 DEBUG [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1351): Waiting on 96b172c55f3e7fec396050c330fc9f7b 2024-11-12T07:53:19,142 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/96b172c55f3e7fec396050c330fc9f7b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-12T07:53:19,143 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. 2024-11-12T07:53:19,143 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 96b172c55f3e7fec396050c330fc9f7b: Waiting for close lock at 1731397998692Running coprocessor pre-close hooks at 1731397998692Disabling compacts and flushes for region at 1731397998692Disabling writes for close at 1731397998692Obtaining lock to block concurrent updates at 1731397998692Preparing flush snapshotting stores in 96b172c55f3e7fec396050c330fc9f7b at 1731397998692Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731397998693 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. at 1731397998693Flushing 96b172c55f3e7fec396050c330fc9f7b/testFamily: creating writer at 1731397998693Flushing 96b172c55f3e7fec396050c330fc9f7b/testFamily: appending metadata at 1731397998707 (+14 ms)Flushing 96b172c55f3e7fec396050c330fc9f7b/testFamily: closing flushed file at 1731397998707Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f56ac48: reopening flushed file at 1731397999127 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 96b172c55f3e7fec396050c330fc9f7b in 445ms, sequenceid=8, compaction requested=false at 1731397999137 (+10 ms)Writing region close event to WAL at 1731397999138 (+1 ms)Running coprocessor post-close hooks at 1731397999143 (+5 ms)Closed at 1731397999143 2024-11-12T07:53:19,143 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731397970582.96b172c55f3e7fec396050c330fc9f7b. 2024-11-12T07:53:19,338 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(976): stopping server f9fc96254ef8,43507,1731397896042; all regions closed. 2024-11-12T07:53:19,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741834_1010 (size=19479) 2024-11-12T07:53:19,354 DEBUG [RS:0;f9fc96254ef8:43507 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/oldWALs 2024-11-12T07:53:19,354 INFO [RS:0;f9fc96254ef8:43507 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f9fc96254ef8%2C43507%2C1731397896042.meta:.meta(num 1731397929421) 2024-11-12T07:53:19,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741833_1009 (size=20031) 2024-11-12T07:53:19,359 DEBUG [RS:0;f9fc96254ef8:43507 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/oldWALs 2024-11-12T07:53:19,359 INFO [RS:0;f9fc96254ef8:43507 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f9fc96254ef8%2C43507%2C1731397896042:(num 1731397928918) 2024-11-12T07:53:19,359 DEBUG [RS:0;f9fc96254ef8:43507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:53:19,359 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T07:53:19,359 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T07:53:19,359 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.ChoreService(370): Chore service for: regionserver/f9fc96254ef8:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T07:53:19,360 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T07:53:19,360 INFO [regionserver/f9fc96254ef8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T07:53:19,361 INFO [RS:0;f9fc96254ef8:43507 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43507 2024-11-12T07:53:19,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f9fc96254ef8,43507,1731397896042 2024-11-12T07:53:19,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T07:53:19,377 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T07:53:19,386 INFO [regionserver/f9fc96254ef8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T07:53:19,394 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f9fc96254ef8,43507,1731397896042] 2024-11-12T07:53:19,410 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f9fc96254ef8,43507,1731397896042 already deleted, retry=false 2024-11-12T07:53:19,410 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f9fc96254ef8,43507,1731397896042 expired; onlineServers=0 2024-11-12T07:53:19,410 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f9fc96254ef8,42363,1731397895269' ***** 2024-11-12T07:53:19,410 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T07:53:19,410 INFO [M:0;f9fc96254ef8:42363 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T07:53:19,410 INFO [M:0;f9fc96254ef8:42363 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T07:53:19,410 DEBUG [M:0;f9fc96254ef8:42363 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T07:53:19,411 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T07:53:19,411 DEBUG [M:0;f9fc96254ef8:42363 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T07:53:19,411 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.small.0-1731397898687 {}] cleaner.HFileCleaner(306): Exit Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.small.0-1731397898687,5,FailOnTimeoutGroup] 2024-11-12T07:53:19,411 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.large.0-1731397898683 {}] cleaner.HFileCleaner(306): Exit Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.large.0-1731397898683,5,FailOnTimeoutGroup] 2024-11-12T07:53:19,411 INFO [M:0;f9fc96254ef8:42363 {}] hbase.ChoreService(370): Chore service for: master/f9fc96254ef8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T07:53:19,411 INFO [M:0;f9fc96254ef8:42363 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T07:53:19,411 DEBUG [M:0;f9fc96254ef8:42363 {}] master.HMaster(1795): Stopping service threads 2024-11-12T07:53:19,411 INFO [M:0;f9fc96254ef8:42363 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T07:53:19,411 INFO [M:0;f9fc96254ef8:42363 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T07:53:19,412 INFO [M:0;f9fc96254ef8:42363 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T07:53:19,412 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T07:53:19,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T07:53:19,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:53:19,418 DEBUG [M:0;f9fc96254ef8:42363 {}] zookeeper.ZKUtil(347): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T07:53:19,418 WARN [M:0;f9fc96254ef8:42363 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T07:53:19,419 INFO [M:0;f9fc96254ef8:42363 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/.lastflushedseqids 2024-11-12T07:53:19,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741868_1044 (size=649) 2024-11-12T07:53:19,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:53:19,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43507-0x1012f50d3b20001, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:53:19,495 INFO [RS:0;f9fc96254ef8:43507 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T07:53:19,496 INFO [RS:0;f9fc96254ef8:43507 {}] regionserver.HRegionServer(1031): Exiting; stopping=f9fc96254ef8,43507,1731397896042; zookeeper connection closed. 2024-11-12T07:53:19,497 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@a3e6532 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@a3e6532 2024-11-12T07:53:19,498 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-12T07:53:19,832 INFO [M:0;f9fc96254ef8:42363 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T07:53:19,833 INFO [M:0;f9fc96254ef8:42363 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T07:53:19,833 DEBUG [M:0;f9fc96254ef8:42363 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T07:53:19,834 INFO [M:0;f9fc96254ef8:42363 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:53:19,834 DEBUG [M:0;f9fc96254ef8:42363 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:53:19,834 DEBUG [M:0;f9fc96254ef8:42363 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T07:53:19,834 DEBUG [M:0;f9fc96254ef8:42363 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:53:19,835 INFO [M:0;f9fc96254ef8:42363 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.97 KB heapSize=190.87 KB 2024-11-12T07:53:19,856 DEBUG [M:0;f9fc96254ef8:42363 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbfed8a2993a4c7b9eba53e96cb0c698 is 82, key is hbase:meta,,1/info:regioninfo/1731397929516/Put/seqid=0 2024-11-12T07:53:19,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741869_1045 (size=5672) 2024-11-12T07:53:20,264 INFO [M:0;f9fc96254ef8:42363 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbfed8a2993a4c7b9eba53e96cb0c698 2024-11-12T07:53:20,299 DEBUG [M:0;f9fc96254ef8:42363 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c12809d1f4e346ea9a65e7fc7e420ce8 is 811, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731397931121/Put/seqid=0 2024-11-12T07:53:20,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741870_1046 (size=14068) 2024-11-12T07:53:20,575 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:53:20,704 INFO [M:0;f9fc96254ef8:42363 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.41 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c12809d1f4e346ea9a65e7fc7e420ce8 2024-11-12T07:53:20,712 INFO [M:0;f9fc96254ef8:42363 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c12809d1f4e346ea9a65e7fc7e420ce8 2024-11-12T07:53:20,735 DEBUG [M:0;f9fc96254ef8:42363 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/48fd07ed203e4ecd93428cd1aaef89d0 is 69, key is f9fc96254ef8,43507,1731397896042/rs:state/1731397898734/Put/seqid=0 2024-11-12T07:53:20,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741871_1047 (size=5156) 2024-11-12T07:53:21,141 INFO [M:0;f9fc96254ef8:42363 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/48fd07ed203e4ecd93428cd1aaef89d0 2024-11-12T07:53:21,156 DEBUG [M:0;f9fc96254ef8:42363 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbfed8a2993a4c7b9eba53e96cb0c698 as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bbfed8a2993a4c7b9eba53e96cb0c698 2024-11-12T07:53:21,163 INFO [M:0;f9fc96254ef8:42363 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bbfed8a2993a4c7b9eba53e96cb0c698, entries=8, sequenceid=317, filesize=5.5 K 2024-11-12T07:53:21,164 DEBUG [M:0;f9fc96254ef8:42363 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c12809d1f4e346ea9a65e7fc7e420ce8 as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c12809d1f4e346ea9a65e7fc7e420ce8 2024-11-12T07:53:21,170 INFO [M:0;f9fc96254ef8:42363 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c12809d1f4e346ea9a65e7fc7e420ce8 2024-11-12T07:53:21,171 INFO [M:0;f9fc96254ef8:42363 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c12809d1f4e346ea9a65e7fc7e420ce8, entries=33, sequenceid=317, filesize=13.7 K 2024-11-12T07:53:21,172 DEBUG [M:0;f9fc96254ef8:42363 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/48fd07ed203e4ecd93428cd1aaef89d0 as hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/48fd07ed203e4ecd93428cd1aaef89d0 2024-11-12T07:53:21,178 INFO [M:0;f9fc96254ef8:42363 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40941/user/jenkins/test-data/65ec3ebc-296d-5ca8-47e9-27f2e14a077a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/48fd07ed203e4ecd93428cd1aaef89d0, entries=1, sequenceid=317, filesize=5.0 K 2024-11-12T07:53:21,179 INFO [M:0;f9fc96254ef8:42363 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.97 KB/166879, heapSize ~190.57 KB/195144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1345ms, sequenceid=317, compaction requested=false 2024-11-12T07:53:21,180 INFO [M:0;f9fc96254ef8:42363 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:53:21,180 DEBUG [M:0;f9fc96254ef8:42363 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731397999833Disabling compacts and flushes for region at 1731397999833Disabling writes for close at 1731397999834 (+1 ms)Obtaining lock to block concurrent updates at 1731397999835 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731397999835Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166879, getHeapSize=195384, getOffHeapSize=0, getCellsCount=364 at 1731397999835Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731397999836 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731397999837 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731397999855 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731397999855Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731398000278 (+423 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731398000298 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731398000298Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731398000713 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731398000734 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731398000734Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12ae8eb: reopening flushed file at 1731398001155 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10fbff5c: reopening flushed file at 1731398001163 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@695889ac: reopening flushed file at 1731398001171 (+8 ms)Finished flush of dataSize ~162.97 KB/166879, heapSize ~190.57 KB/195144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1345ms, sequenceid=317, compaction requested=false at 1731398001179 (+8 ms)Writing region close event to WAL at 1731398001180 (+1 ms)Closed at 1731398001180 2024-11-12T07:53:21,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39471 is added to blk_1073741830_1006 (size=188091) 2024-11-12T07:53:21,184 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T07:53:21,184 INFO [M:0;f9fc96254ef8:42363 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T07:53:21,184 INFO [M:0;f9fc96254ef8:42363 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42363 2024-11-12T07:53:21,185 INFO [M:0;f9fc96254ef8:42363 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T07:53:21,299 INFO [M:0;f9fc96254ef8:42363 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T07:53:21,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:53:21,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42363-0x1012f50d3b20000, quorum=127.0.0.1:49256, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:53:21,303 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5b0930ac{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T07:53:21,305 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@257dd063{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T07:53:21,305 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T07:53:21,306 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d71513b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T07:53:21,306 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b440b02{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/hadoop.log.dir/,STOPPED} 2024-11-12T07:53:21,308 WARN [BP-945294674-172.17.0.2-1731397891410 heartbeating to localhost/127.0.0.1:40941 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T07:53:21,308 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T07:53:21,308 WARN [BP-945294674-172.17.0.2-1731397891410 heartbeating to localhost/127.0.0.1:40941 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-945294674-172.17.0.2-1731397891410 (Datanode Uuid ca631ca4-2819-4a29-b7dd-8ab31d1fd8e0) service to localhost/127.0.0.1:40941 2024-11-12T07:53:21,308 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T07:53:21,309 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/cluster_af29951c-f6f7-2c1e-a0a4-3fa84ccae668/data/data1/current/BP-945294674-172.17.0.2-1731397891410 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T07:53:21,309 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/cluster_af29951c-f6f7-2c1e-a0a4-3fa84ccae668/data/data2/current/BP-945294674-172.17.0.2-1731397891410 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T07:53:21,309 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T07:53:21,318 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@b5692ae{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T07:53:21,319 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1809bc91{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T07:53:21,319 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T07:53:21,319 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@549fd2eb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T07:53:21,319 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@24268d3a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4130d3ac-e62c-db56-8803-b589fe41d34c/hadoop.log.dir/,STOPPED} 2024-11-12T07:53:21,326 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T07:53:21,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down